<!DOCTYPE html>
<html lang="zh-CN">

<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
  <meta name="theme-color" content="#222">
  <meta name="generator" content="Hexo 4.2.1">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png">
  <link rel="mask-icon" href="/images/safari-pinned-tab.svg" color="#222">
  <link rel="stylesheet" href="/css/main.css">
  <link rel="stylesheet" href="/lib/font-awesome/css/all.min.css">
  <link rel="stylesheet" href="/lib/pace/pace-theme-minimal.min.css">
  <script src="/lib/pace/pace.min.js"></script>
  <script id="hexo-configurations">
    var NexT = window.NexT ||
    {};
    var CONFIG = {
      "hostname": "cuiqingcai.com",
      "root": "/",
      "scheme": "Pisces",
      "version": "7.8.0",
      "exturl": false,
      "sidebar":
      {
        "position": "right",
        "width": 360,
        "display": "post",
        "padding": 18,
        "offset": 12,
        "onmobile": false,
        "widgets": [
          {
            "type": "image",
            "name": "阿布云",
            "enable": false,
            "url": "https://www.abuyun.com/http-proxy/introduce.html",
            "src": "https://qiniu.cuiqingcai.com/88au8.jpg",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "天验",
            "enable": true,
            "url": "https://tutorial.lengyue.video/?coupon=12ef4b1a-a3db-11ea-bb37-0242ac130002_cqx_850",
            "src": "https://qiniu.cuiqingcai.com/bco2a.png",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "华为云",
            "enable": false,
            "url": "https://activity.huaweicloud.com/2020_618_promotion/index.html?bpName=5f9f98a29e2c40b780c1793086f29fe2&bindType=1&salesID=wangyubei",
            "src": "https://qiniu.cuiqingcai.com/y42ik.jpg",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "张小鸡",
            "enable": false,
            "url": "http://www.zxiaoji.com/",
            "src": "https://qiniu.cuiqingcai.com/fm72f.png",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "Luminati",
            "src": "https://qiniu.cuiqingcai.com/ikkq9.jpg",
            "url": "https://luminati-china.io/?affiliate=ref_5fbbaaa9647883f5c6f77095",
            "width": "100%",
            "enable": false
      },
          {
            "type": "image",
            "name": "IPIDEA",
            "url": "http://www.ipidea.net/?utm-source=cqc&utm-keyword=?cqc",
            "src": "https://qiniu.cuiqingcai.com/0ywun.png",
            "width": "100%",
            "enable": true
      },
          {
            "type": "tags",
            "name": "标签云",
            "enable": true
      },
          {
            "type": "categories",
            "name": "分类",
            "enable": true
      },
          {
            "type": "friends",
            "name": "友情链接",
            "enable": true
      },
          {
            "type": "hot",
            "name": "猜你喜欢",
            "enable": true
      }]
      },
      "copycode":
      {
        "enable": true,
        "show_result": true,
        "style": "mac"
      },
      "back2top":
      {
        "enable": true,
        "sidebar": false,
        "scrollpercent": true
      },
      "bookmark":
      {
        "enable": false,
        "color": "#222",
        "save": "auto"
      },
      "fancybox": false,
      "mediumzoom": false,
      "lazyload": false,
      "pangu": true,
      "comments":
      {
        "style": "tabs",
        "active": "gitalk",
        "storage": true,
        "lazyload": false,
        "nav": null,
        "activeClass": "gitalk"
      },
      "algolia":
      {
        "hits":
        {
          "per_page": 10
        },
        "labels":
        {
          "input_placeholder": "Search for Posts",
          "hits_empty": "We didn't find any results for the search: ${query}",
          "hits_stats": "${hits} results found in ${time} ms"
        }
      },
      "localsearch":
      {
        "enable": true,
        "trigger": "auto",
        "top_n_per_article": 10,
        "unescape": false,
        "preload": false
      },
      "motion":
      {
        "enable": false,
        "async": false,
        "transition":
        {
          "post_block": "bounceDownIn",
          "post_header": "slideDownIn",
          "post_body": "slideDownIn",
          "coll_header": "slideLeftIn",
          "sidebar": "slideUpIn"
        }
      },
      "path": "search.xml"
    };

  </script>
  <meta name="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
  <meta property="og:type" content="website">
  <meta property="og:title" content="静觅">
  <meta property="og:url" content="https://cuiqingcai.com/page/16/index.html">
  <meta property="og:site_name" content="静觅">
  <meta property="og:description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
  <meta property="og:locale" content="zh_CN">
  <meta property="article:author" content="崔庆才">
  <meta property="article:tag" content="崔庆才">
  <meta property="article:tag" content="静觅">
  <meta property="article:tag" content="PHP">
  <meta property="article:tag" content="Java">
  <meta property="article:tag" content="Python">
  <meta property="article:tag" content="Spider">
  <meta property="article:tag" content="爬虫">
  <meta property="article:tag" content="Web">
  <meta property="article:tag" content="Kubernetes">
  <meta property="article:tag" content="深度学习">
  <meta property="article:tag" content="机器学习">
  <meta property="article:tag" content="数据分析">
  <meta property="article:tag" content="网络">
  <meta property="article:tag" content="IT">
  <meta property="article:tag" content="技术">
  <meta property="article:tag" content="博客">
  <meta name="twitter:card" content="summary">
  <link rel="canonical" href="https://cuiqingcai.com/page/16/">
  <script id="page-configurations">
    // https://hexo.io/docs/variables.html
    CONFIG.page = {
      sidebar: "",
      isHome: true,
      isPost: false,
      lang: 'zh-CN'
    };

  </script>
  <title>静觅丨崔庆才的个人站点</title>
  <meta name="google-site-verification" content="p_bIcnvirkFzG2dYKuNDivKD8-STet5W7D-01woA2fc" />
  <noscript>
    <style>
      .use-motion .brand,
      .use-motion .menu-item,
      .sidebar-inner,
      .use-motion .post-block,
      .use-motion .pagination,
      .use-motion .comments,
      .use-motion .post-header,
      .use-motion .post-body,
      .use-motion .collection-header
      {
        opacity: initial;
      }

      .use-motion .site-title,
      .use-motion .site-subtitle
      {
        opacity: initial;
        top: initial;
      }

      .use-motion .logo-line-before i
      {
        left: initial;
      }

      .use-motion .logo-line-after i
      {
        right: initial;
      }

    </style>
  </noscript>
  <link rel="alternate" href="/atom.xml" title="静觅" type="application/atom+xml">
</head>

<body itemscope itemtype="http://schema.org/WebPage">
  <div class="container">
    <div class="headband"></div>
    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner">
        <div class="site-brand-container">
          <div class="site-nav-toggle">
            <div class="toggle" aria-label="切换导航栏">
              <span class="toggle-line toggle-line-first"></span>
              <span class="toggle-line toggle-line-middle"></span>
              <span class="toggle-line toggle-line-last"></span>
            </div>
          </div>
          <div class="site-meta">
            <a href="/" class="brand" rel="start">
              <span class="logo-line-before"><i></i></span>
              <h1 class="site-title">静觅 <span class="site-subtitle"> 崔庆才的个人站点 </span>
              </h1>
              <span class="logo-line-after"><i></i></span>
            </a>
          </div>
          <div class="site-nav-right">
            <div class="toggle popup-trigger">
              <i class="fa fa-search fa-fw fa-lg"></i>
            </div>
          </div>
        </div>
        <nav class="site-nav">
          <ul id="menu" class="main-menu menu">
            <li class="menu-item menu-item-home">
              <a href="/" rel="section">首页</a>
            </li>
            <li class="menu-item menu-item-archives">
              <a href="/archives/" rel="section">文章列表</a>
            </li>
            <li class="menu-item menu-item-tags">
              <a href="/tags/" rel="section">文章标签</a>
            </li>
            <li class="menu-item menu-item-categories">
              <a href="/categories/" rel="section">文章分类</a>
            </li>
            <li class="menu-item menu-item-about">
              <a href="/about/" rel="section">关于博主</a>
            </li>
            <li class="menu-item menu-item-message">
              <a href="/message/" rel="section">给我留言</a>
            </li>
            <li class="menu-item menu-item-search">
              <a role="button" class="popup-trigger">搜索 </a>
            </li>
          </ul>
        </nav>
        <div class="search-pop-overlay">
          <div class="popup search-popup">
            <div class="search-header">
              <span class="search-icon">
                <i class="fa fa-search"></i>
              </span>
              <div class="search-input-container">
                <input autocomplete="off" autocapitalize="off" placeholder="搜索..." spellcheck="false" type="search" class="search-input">
              </div>
              <span class="popup-btn-close">
                <i class="fa fa-times-circle"></i>
              </span>
            </div>
            <div id="search-result">
              <div id="no-result">
                <i class="fa fa-spinner fa-pulse fa-5x fa-fw"></i>
              </div>
            </div>
          </div>
        </div>
      </div>
    </header>
    <div class="back-to-top">
      <i class="fa fa-arrow-up"></i>
      <span>0%</span>
    </div>
    <div class="reading-progress-bar"></div>
    <main class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div class="content index posts-expand">
            <div class="carousel">
              <div id="wowslider-container">
                <div class="ws_images">
                  <ul>
                    <li><a target="_blank" href="https://cuiqingcai.com/5052.html"><img title="Python3网络爬虫开发实战教程" src="https://qiniu.cuiqingcai.com/ipy96.jpg" /></a></li>
                    <li><a target="_blank" href="https://t.lagou.com/fRCBRsRCSN6FA"><img title="52讲轻松搞定网络爬虫" src="https://qiniu.cuiqingcai.com/fqq5e.png" /></a></li>
                    <li><a target="_blank" href="https://brightdata.grsm.io/cuiqingcai"><img title="亮网络解锁器" src="https://qiniu.cuiqingcai.com/6qnb7.png" /></a></li>
                    <li><a target="_blank" href="https://cuiqingcai.com/4320.html"><img title="Python3网络爬虫开发视频教程" src="https://qiniu.cuiqingcai.com/bjrny.jpg" /></a></li>
                    <li><a target="_blank" href="https://cuiqingcai.com/5094.html"><img title="爬虫代理哪家强？十大付费代理详细对比评测出炉！" src="https://qiniu.cuiqingcai.com/nifs6.jpg" /></a></li>
                  </ul>
                </div>
                <div class="ws_thumbs">
                  <div>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/ipy96.jpg" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/fqq5e.png" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/6qnb7.png" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/bjrny.jpg" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/nifs6.jpg" /></a>
                  </div>
                </div>
                <div class="ws_shadow"></div>
              </div>
            </div>
            <link rel="stylesheet" href="/lib/wowslide/slide.css">
            <script src="/lib/wowslide/jquery.min.js"></script>
            <script src="/lib/wowslide/slider.js"></script>
            <script>
              jQuery("#wowslider-container").wowSlider(
              {
                effect: "cube",
                prev: "",
                next: "",
                duration: 20 * 100,
                delay: 20 * 100,
                width: 716,
                height: 297,
                autoPlay: true,
                playPause: true,
                stopOnHover: false,
                loop: false,
                bullets: 0,
                caption: true,
                captionEffect: "slide",
                controls: true,
                onBeforeStep: 0,
                images: 0
              });

            </script>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5844.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5844.html" class="post-title-link" itemprop="url">中文分词原理及工具</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="原理"><a href="#原理" class="headerlink" title="原理"></a>原理</h2>
                  <p>中文分词，即 Chinese Word Segmentation，即将一个汉字序列进行切分，得到一个个单独的词。表面上看，分词其实就是那么回事，但分词效果好不好对信息检索、实验结果还是有很大影响的，同时分词的背后其实是涉及各种各样的算法的。 中文分词与英文分词有很大的不同，对英文而言，一个单词就是一个词，而汉语是以字为基本的书写单位，词语之间没有明显的区分标记，需要人为切分。根据其特点，可以把分词算法分为四大类：</p>
                  <ul>
                    <li>基于规则的分词方法</li>
                    <li>基于统计的分词方法</li>
                    <li>基于语义的分词方法</li>
                    <li>基于理解的分词方法</li>
                  </ul>
                  <p>下面我们对这几种方法分别进行总结。</p>
                  <h3 id="基于规则的分词方法"><a href="#基于规则的分词方法" class="headerlink" title="基于规则的分词方法"></a>基于规则的分词方法</h3>
                  <p>这种方法又叫作机械分词方法、基于字典的分词方法，它是按照一定的策略将待分析的汉字串与一个“充分大的”机器词典中的词条进行匹配。若在词典中找到某个字符串，则匹配成功。该方法有三个要素，即分词词典、文本扫描顺序和匹配原则。文本的扫描顺序有正向扫描、逆向扫描和双向扫描。匹配原则主要有最大匹配、最小匹配、逐词匹配和最佳匹配。</p>
                  <ul>
                    <li>最大匹配法（MM）。基本思想是：假设自动分词词典中的最长词条所含汉字的个数为 i，则取被处理材料当前字符串序列中的前 i 个字符作为匹配字段，查找分词词典，若词典中有这样一个 i 字词，则匹配成功，匹配字段作为一个词被切分出来；若词典中找不到这样的一个 i 字词，则匹配失败，匹配字段去掉最后一个汉字，剩下的字符作为新的匹配字段，再进行匹配，如此进行下去，直到匹配成功为止。统计结果表明，该方法的错误率 为 1/169。</li>
                    <li>逆向最大匹配法（RMM）。该方法的分词过程与 MM 法相同，不同的是从句子（或文章）末尾开始处理，每次匹配不成功时去掉的是前面的一个汉字。统计结果表明，该方法的错误率为 1/245。</li>
                    <li>逐词遍历法。把词典中的词按照由长到短递减的顺序逐字搜索整个待处理的材料，一直到把全部的词切分出来为止。不论分词词典多大，被处理的材料多么小，都得把这个分词词典匹配一遍。</li>
                    <li>设立切分标志法。切分标志有自然和非自然之分。自然切分标志是指文章中出现的非文字符号，如标点符号等；非自然标志是利用词缀和不构成词的词（包 括单音词、复音节词以及象声词等）。设立切分标志法首先收集众多的切分标志，分词时先找出切分标志，把句子切分为一些较短的字段，再用 MM、RMM 或其它的方法进行细加工。这种方法并非真正意义上的分词方法，只是自动分词的一种前处理方式而已，它要额外消耗时间扫描切分标志，增加存储空间存放那些非 自然切分标志。</li>
                    <li>最佳匹配法（OM）。此法分为正向的最佳匹配法和逆向的最佳匹配法，其出发点是：在词典中按词频的大小顺序排列词条，以求缩短对分词词典的检索时 间，达到最佳效果，从而降低分词的时间复杂度，加快分词速度。实质上，这种方法也不是一种纯粹意义上的分词方法，它只是一种对分词词典的组织方式。OM 法的分词词典每条词的前面必须有指明长度的数据项，所以其空间复杂度有所增加，对提高分词精度没有影响，分词处理的时间复杂度有所降低。</li>
                  </ul>
                  <p>此种方法优点是简单，易于实现。但缺点有很多：匹配速度慢；存在交集型和组合型歧义切分问题；词本身没有一个标准的定义，没有统一标准的词集；不同词典产生的歧义也不同；缺乏自学习的智能性。</p>
                  <h3 id="基于统计的分词方法"><a href="#基于统计的分词方法" class="headerlink" title="基于统计的分词方法"></a>基于统计的分词方法</h3>
                  <p>该方法的主要思想：词是稳定的组合，因此在上下文中，相邻的字同时出现的次数越多，就越有可能构成一个词。因此字与字相邻出现的概率或频率能较好地反映成词的可信度。可以对训练文本中相邻出现的各个字的组合的频度进行统计，计算它们之间的互现信息。互现信息体现了汉字之间结合关系的紧密程度。当紧密程 度高于某一个阈值时，便可以认为此字组可能构成了一个词。该方法又称为无字典分词。 该方法所应用的主要的统计模型有：N 元文法模型（N-gram）、隐马尔可夫模型（Hiden Markov Model，HMM）、最大熵模型（ME）、条件随机场模型（Conditional Random Fields，CRF）等。 在实际应用中此类分词算法一般是将其与基于词典的分词方法结合起来，既发挥匹配分词切分速度快、效率高的特点，又利用了无词典分词结合上下文识别生词、自动消除歧义的优点。</p>
                  <h3 id="基于语义的分词方法"><a href="#基于语义的分词方法" class="headerlink" title="基于语义的分词方法"></a>基于语义的分词方法</h3>
                  <p>语义分词法引入了语义分析，对自然语言自身的语言信息进行更多的处理，如扩充转移网络法、知识分词语义分析法、邻接约束法、综合匹配法、后缀分词法、特征词库法、矩阵约束法、语法分析法等。</p>
                  <ul>
                    <li>扩充转移网络法。该方法以有限状态机概念为基础。有限状态机只能识别正则语言，对有限状态机作的第一次扩充使其具有递归能力，形成递归转移网络 （RTN）。在RTN 中，弧线上的标志不仅可以是终极符（语言中的单词）或非终极符（词类），还可以调用另外的子网络名字分非终极符（如字或字串的成词条件）。这样，计算机在 运行某个子网络时，就可以调用另外的子网络，还可以递归调用。词法扩充转移网络的使用， 使分词处理和语言理解的句法处理阶段交互成为可能，并且有效地解决了汉语分词的歧义。</li>
                    <li>矩阵约束法。其基本思想是：先建立一个语法约束矩阵和一个语义约束矩阵， 其中元素分别表明具有某词性的词和具有另一词性的词相邻是否符合语法规则， 属于某语义类的词和属于另一词义类的词相邻是否符合逻辑，机器在切分时以之约束分词结果。</li>
                  </ul>
                  <h3 id="基于理解的分词方法"><a href="#基于理解的分词方法" class="headerlink" title="基于理解的分词方法"></a>基于理解的分词方法</h3>
                  <p>基于理解的分词方法是通过让计算机模拟人对句子的理解，达到识别词的效果。其基本思想就是在分词的同时进行句法、语义分析，利用句法信息和语义信息来处理歧义现象。它通常包括三个部分：分词子系统、句法语义子系统、总控部分。在总控部分的协调下，分词子系统可以获得有关词、句子等的句法和语义信息来对分词歧义进行判断，即它模拟了人对句子的理解过程。这种分词方法需要使用大量的语言知识和信息。目前基于理解的分词方法主要有专家系统分词法和神经网络分词法等。</p>
                  <ul>
                    <li>专家系统分词法。从专家系统角度把分词的知识（包括常识性分词知识与消除歧义切分的启发性知识即歧义切分规则）从实现分词过程的推理机中独立出来，使知识库的维护与推理机的实现互不干扰，从而使知识库易于维护和管理。它还具有发现交集歧义字段和多义组合歧义字段的能力和一定的自学习功能。</li>
                    <li>神经网络分词法。该方法是模拟人脑并行，分布处理和建立数值计算模型工作的。它将分词知识所分散隐式的方法存入神经网络内部，通过自学习和训练修改内部权值，以达到正确的分词结果，最后给出神经网络自动分词结果，如使用 LSTM、GRU 等神经网络模型等。</li>
                    <li>神经网络专家系统集成式分词法。该方法首先启动神经网络进行分词，当神经网络对新出现的词不能给出准确切分时，激活专家系统进行分析判断，依据知识库进行推理，得出初步分析，并启动学习机制对神经网络进行训练。该方法可以较充分发挥神经网络与专家系统二者优势，进一步提高分词效率。</li>
                  </ul>
                  <p>以上便是对分词算法的基本介绍，接下来我们再介绍几个比较实用的分词 Python 库及它们的使用方法。</p>
                  <h2 id="分词工具"><a href="#分词工具" class="headerlink" title="分词工具"></a>分词工具</h2>
                  <p>在这里介绍几个比较有代表性的支持分词的 Python 库，主要有：</p>
                  <h3 id="1-jieba"><a href="#1-jieba" class="headerlink" title="1. jieba"></a>1. jieba</h3>
                  <p>专用于分词的 Python 库，GitHub：<a href="https://github.com/fxsjy/jieba" target="_blank" rel="noopener">https://github.com/fxsjy/jieba</a>，分词效果较好。 支持三种分词模式：</p>
                  <ul>
                    <li>精确模式，试图将句子最精确地切开，适合文本分析。</li>
                    <li>全模式，将句子中所有的可能成词的词语都扫描出来，速度非常快，但是不能解决歧义。</li>
                    <li>搜索引擎模式：在精确模式的基础上，对长词再次切分，提高召回率，适用于搜索引擎分词。</li>
                  </ul>
                  <p>另外 jieba 支持繁体分词，支持自定义词典。 其使用的算法是基于统计的分词方法，主要有如下几种：</p>
                  <ul>
                    <li>基于前缀词典实现高效的词图扫描，生成句子中汉字所有可能成词情况所构成的有向无环图 (DAG)</li>
                    <li>采用了动态规划查找最大概率路径, 找出基于词频的最大切分组合</li>
                    <li>对于未登录词，采用了基于汉字成词能力的 HMM 模型，使用了 Viterbi 算法</li>
                  </ul>
                  <h4 id="精确模式分词"><a href="#精确模式分词" class="headerlink" title="精确模式分词"></a>精确模式分词</h4>
                  <p>首先我们来看下精确模式分词，使用 lcut() 方法，类似 cut() 方法，其参数和 cut() 是一致的，只不过返回结果是列表而不是生成器，默认使用精确模式，代码如下：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import jieba</span><br><span class="line"><span class="keyword">string</span> = <span class="string">'这个把手该换了，我不喜欢日本和服，别把手放在我的肩膀上，工信处女干事每月经过下属科室都要亲口交代24口交换机等技术性器件的安装工作'</span></span><br><span class="line"><span class="built_in">result</span> = jieba.lcut(<span class="keyword">string</span>)</span><br><span class="line">print(<span class="built_in">len</span>(<span class="built_in">result</span>), <span class="string">'/'</span>.join(<span class="built_in">result</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">38</span> 这个<span class="regexp">/把手/</span>该换<span class="regexp">/了/</span>，<span class="regexp">/我/</span>不<span class="regexp">/喜欢/</span>日本<span class="regexp">/和服/</span>，<span class="regexp">/别/</span>把手<span class="regexp">/放在/</span>我<span class="regexp">/的/</span>肩膀<span class="regexp">/上/</span>，<span class="regexp">/工信处/</span>女干事<span class="regexp">/每月/</span>经过<span class="regexp">/下属/</span>科室<span class="regexp">/都/</span>要<span class="regexp">/亲口/</span>交代<span class="regexp">/24/</span>口<span class="regexp">/交换机/</span>等<span class="regexp">/技术性/</span>器件<span class="regexp">/的/</span>安装<span class="regexp">/工作</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可见分词效果还是不错的。</p>
                  <h4 id="全模式分词"><a href="#全模式分词" class="headerlink" title="全模式分词"></a>全模式分词</h4>
                  <p>使用全模式分词需要添加 cut_all 参数，将其设置为 True，代码如下：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">result</span> = jieba.lcut(<span class="keyword">string</span>, cut_all=True)</span><br><span class="line">print(<span class="built_in">len</span>(<span class="built_in">result</span>), <span class="string">'/'</span>.join(<span class="built_in">result</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">51</span> 这个<span class="regexp">/把手/</span>该换<span class="regexp">/了/</span><span class="regexp">//</span>我<span class="regexp">/不/</span>喜欢<span class="regexp">/日本/</span>和服<span class="regexp">//</span><span class="regexp">/别/</span>把手<span class="regexp">/放在/</span>我<span class="regexp">/的/</span>肩膀<span class="regexp">/上/</span><span class="regexp">//</span>工信处<span class="regexp">/处女/</span>女干事<span class="regexp">/干事/</span>每月<span class="regexp">/月经/</span>经过<span class="regexp">/下属/</span>科室<span class="regexp">/都/</span>要<span class="regexp">/亲口/</span>口交<span class="regexp">/交代/</span><span class="number">24</span><span class="regexp">/口交/</span>交换<span class="regexp">/交换机/</span>换机<span class="regexp">/等/</span>技术<span class="regexp">/技术性/</span>性器<span class="regexp">/器件/</span>的<span class="regexp">/安装/</span>安装工<span class="regexp">/装工/</span>工作</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h4 id="搜索引擎模式分词"><a href="#搜索引擎模式分词" class="headerlink" title="搜索引擎模式分词"></a>搜索引擎模式分词</h4>
                  <p>使用搜索引擎模式分词需要调用 cut_for_search() 方法，代码如下：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">result</span> = jieba.lcut_for_search(<span class="keyword">string</span>)</span><br><span class="line">print(<span class="built_in">len</span>(<span class="built_in">result</span>), <span class="string">'/'</span>.join(<span class="built_in">result</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">42</span> 这个<span class="regexp">/把手/</span>该换<span class="regexp">/了/</span>，<span class="regexp">/我/</span>不<span class="regexp">/喜欢/</span>日本<span class="regexp">/和服/</span>，<span class="regexp">/别/</span>把手<span class="regexp">/放在/</span>我<span class="regexp">/的/</span>肩膀<span class="regexp">/上/</span>，<span class="regexp">/工信处/</span>干事<span class="regexp">/女干事/</span>每月<span class="regexp">/经过/</span>下属<span class="regexp">/科室/</span>都<span class="regexp">/要/</span>亲口<span class="regexp">/交代/</span><span class="number">24</span><span class="regexp">/口/</span>交换<span class="regexp">/换机/</span>交换机<span class="regexp">/等/</span>技术<span class="regexp">/技术性/</span>器件<span class="regexp">/的/</span>安装<span class="regexp">/工作</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外可以加入自定义词典，如我们想把 日本和服 作为一个整体，可以把它添加到词典中，代码如下：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">jieba.add_word(<span class="string">'日本和服'</span>)</span><br><span class="line"><span class="built_in">result</span> = jieba.lcut(<span class="keyword">string</span>)</span><br><span class="line">print(<span class="built_in">len</span>(<span class="built_in">result</span>), <span class="string">'/'</span>.join(<span class="built_in">result</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">37</span> 这个<span class="regexp">/把手/</span>该换<span class="regexp">/了/</span>，<span class="regexp">/我/</span>不<span class="regexp">/喜欢/</span>日本和服<span class="regexp">/，/</span>别<span class="regexp">/把手/</span>放在<span class="regexp">/我/</span>的<span class="regexp">/肩膀/</span>上<span class="regexp">/，/</span>工信处<span class="regexp">/女干事/</span>每月<span class="regexp">/经过/</span>下属<span class="regexp">/科室/</span>都<span class="regexp">/要/</span>亲口<span class="regexp">/交代/</span><span class="number">24</span><span class="regexp">/口/</span>交换机<span class="regexp">/等/</span>技术性<span class="regexp">/器件/</span>的<span class="regexp">/安装/</span>工作</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到切分结果中，日本和服 四个字就作为一个整体出现在结果中了，分词数量比精确模式少了一个。</p>
                  <h4 id="词性标注"><a href="#词性标注" class="headerlink" title="词性标注"></a>词性标注</h4>
                  <p>另外 jieba 还支持词性标注，可以输出分词后每个词的词性，实例如下：</p>
                  <figure class="highlight applescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">words</span> = pseg.lcut(<span class="built_in">string</span>)</span><br><span class="line">print(<span class="built_in">list</span>(map(lambda x: <span class="built_in">list</span>(x), <span class="built_in">words</span>)))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[[<span class="symbol">'这个</span>', <span class="symbol">'r</span>'], [<span class="symbol">'把手</span>', <span class="symbol">'v</span>'], [<span class="symbol">'该</span>', <span class="symbol">'r</span>'], [<span class="symbol">'换</span>', <span class="symbol">'v</span>'], [<span class="symbol">'了</span>', <span class="symbol">'ul</span>'], [<span class="symbol">'，</span>', <span class="symbol">'x</span>'], [<span class="symbol">'我</span>', <span class="symbol">'r</span>'], [<span class="symbol">'不</span>', <span class="symbol">'d</span>'], [<span class="symbol">'喜欢</span>', <span class="symbol">'v</span>'], [<span class="symbol">'日本和服</span>', <span class="symbol">'x</span>'], [<span class="symbol">'，</span>', <span class="symbol">'x</span>'], [<span class="symbol">'别</span>', <span class="symbol">'r</span>'], [<span class="symbol">'把手</span>', <span class="symbol">'v</span>'], [<span class="symbol">'放在</span>', <span class="symbol">'v</span>'], [<span class="symbol">'我</span>', <span class="symbol">'r</span>'], [<span class="symbol">'的</span>', <span class="symbol">'uj</span>'], [<span class="symbol">'肩膀</span>', <span class="symbol">'n</span>'], [<span class="symbol">'上</span>', <span class="symbol">'f</span>'], [<span class="symbol">'，</span>', <span class="symbol">'x</span>'], [<span class="symbol">'工信处</span>', <span class="symbol">'n</span>'], [<span class="symbol">'女干事</span>', <span class="symbol">'n</span>'], [<span class="symbol">'每月</span>', <span class="symbol">'r</span>'], [<span class="symbol">'经过</span>', <span class="symbol">'p</span>'], [<span class="symbol">'下属</span>', <span class="symbol">'v</span>'], [<span class="symbol">'科室</span>', <span class="symbol">'n</span>'], [<span class="symbol">'都</span>', <span class="symbol">'d</span>'], [<span class="symbol">'要</span>', <span class="symbol">'v</span>'], [<span class="symbol">'亲口</span>', <span class="symbol">'n</span>'], [<span class="symbol">'交代</span>', <span class="symbol">'n</span>'], [<span class="symbol">'24</span>', <span class="symbol">'m</span>'], [<span class="symbol">'口</span>', <span class="symbol">'n</span>'], [<span class="symbol">'交换机</span>', <span class="symbol">'n</span>'], [<span class="symbol">'等</span>', <span class="symbol">'u</span>'], [<span class="symbol">'技术性</span>', <span class="symbol">'n</span>'], [<span class="symbol">'器件</span>', <span class="symbol">'n</span>'], [<span class="symbol">'的</span>', <span class="symbol">'uj</span>'], [<span class="symbol">'安装</span>', <span class="symbol">'v</span>'], [<span class="symbol">'工作</span>', <span class="symbol">'vn</span>']]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>关于词性的说明可以参考：<a href="https://gist.github.com/luw2007/6016931" target="_blank" rel="noopener">https://gist.github.com/luw2007/6016931</a>。</p>
                  <h3 id="2-SnowNLP"><a href="#2-SnowNLP" class="headerlink" title="2. SnowNLP"></a>2. SnowNLP</h3>
                  <p>SnowNLP: Simplified Chinese Text Processing，可以方便的处理中文文本内容，是受到了 TextBlob 的启发而写的，由于现在大部分的自然语言处理库基本都是针对英文的，于是写了一个方便处理中文的类库，并且和 TextBlob 不同的是，这里没有用 NLTK，所有的算法都是自己实现的，并且自带了一些训练好的字典。GitHub地址：<a href="https://github.com/isnowfy/snownlp" target="_blank" rel="noopener">https://github.com/isnowfy/snownlp</a>。</p>
                  <h4 id="分词"><a href="#分词" class="headerlink" title="分词"></a>分词</h4>
                  <p>这里的分词是基于 Character-Based Generative Model 来实现的，论文地址：<a href="http://aclweb.org/anthology//Y/Y09/Y09-2047.pdf" target="_blank" rel="noopener">http://aclweb.org/anthology//Y/Y09/Y09-2047.pdf</a>，我们还是以上面的例子说明，相关使用说明如下：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">from</span> snownlp import SnowNLP</span><br><span class="line"></span><br><span class="line"><span class="keyword">string</span> = <span class="string">'这个把手该换了，我不喜欢日本和服，别把手放在我的肩膀上，工信处女干事每月经过下属科室都要亲口交代24口交换机等技术性器件的安装工作'</span></span><br><span class="line">s = SnowNLP(<span class="keyword">string</span>)</span><br><span class="line"><span class="built_in">result</span> = s.<span class="keyword">words</span></span><br><span class="line">print(<span class="built_in">len</span>(<span class="built_in">result</span>), <span class="string">'/'</span>.join(<span class="built_in">result</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">40</span> 这个<span class="regexp">/把手/</span>该<span class="regexp">/换/</span>了<span class="regexp">/，/</span>我<span class="regexp">/不/</span>喜欢<span class="regexp">/日本/</span>和<span class="regexp">/服/</span>，<span class="regexp">/别把手/</span>放在<span class="regexp">/我/</span>的<span class="regexp">/肩膀/</span>上<span class="regexp">/，/</span>工<span class="regexp">/信处女/</span>干事<span class="regexp">/每月/</span>经过<span class="regexp">/下属/</span>科室<span class="regexp">/都/</span>要<span class="regexp">/亲口/</span>交代<span class="regexp">/24/</span>口<span class="regexp">/交换机/</span>等<span class="regexp">/技术性/</span>器件<span class="regexp">/的/</span>安装<span class="regexp">/工作</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>经过观察，可以发现分词效果其实不怎么理想，和服 被分开了，工信处 也被分开了，女干事 也被分开了。 另外 SnowNLP 还支持很多功能，例如词性标注（HMM）、情感分析、拼音转换（Trie树）、关键词和摘要生成（TextRank）。 我们简单看一个实例：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="title">print</span><span class="params">(<span class="string">'Tags:'</span>, list(s.tags)</span></span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(<span class="string">'Sentiments:'</span>, s.sentiments)</span></span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(<span class="string">'Pinyin:'</span>, s.pinyin)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">Tags</span>: <span class="selector-attr">[(<span class="string">'这个'</span>, <span class="string">'r'</span>), (<span class="string">'把手'</span>, <span class="string">'Ng'</span>), (<span class="string">'该'</span>, <span class="string">'r'</span>), (<span class="string">'换'</span>, <span class="string">'v'</span>), (<span class="string">'了'</span>, <span class="string">'y'</span>), (<span class="string">'，'</span>, <span class="string">'w'</span>), (<span class="string">'我'</span>, <span class="string">'r'</span>), (<span class="string">'不'</span>, <span class="string">'d'</span>), (<span class="string">'喜欢'</span>, <span class="string">'v'</span>), (<span class="string">'日本'</span>, <span class="string">'ns'</span>), (<span class="string">'和'</span>, <span class="string">'c'</span>), (<span class="string">'服'</span>, <span class="string">'v'</span>), (<span class="string">'，'</span>, <span class="string">'w'</span>), (<span class="string">'别把手'</span>, <span class="string">'ad'</span>), (<span class="string">'放在'</span>, <span class="string">'v'</span>), (<span class="string">'我'</span>, <span class="string">'r'</span>), (<span class="string">'的'</span>, <span class="string">'u'</span>), (<span class="string">'肩膀'</span>, <span class="string">'n'</span>), (<span class="string">'上'</span>, <span class="string">'f'</span>), (<span class="string">'，'</span>, <span class="string">'w'</span>), (<span class="string">'工'</span>, <span class="string">'j'</span>), (<span class="string">'信处女'</span>, <span class="string">'j'</span>), (<span class="string">'干事'</span>, <span class="string">'n'</span>), (<span class="string">'每月'</span>, <span class="string">'r'</span>), (<span class="string">'经过'</span>, <span class="string">'p'</span>), (<span class="string">'下属'</span>, <span class="string">'v'</span>), (<span class="string">'科室'</span>, <span class="string">'n'</span>), (<span class="string">'都'</span>, <span class="string">'d'</span>), (<span class="string">'要'</span>, <span class="string">'v'</span>), (<span class="string">'亲口'</span>, <span class="string">'d'</span>), (<span class="string">'交代'</span>, <span class="string">'v'</span>), (<span class="string">'24'</span>, <span class="string">'m'</span>), (<span class="string">'口'</span>, <span class="string">'q'</span>), (<span class="string">'交换机'</span>, <span class="string">'n'</span>), (<span class="string">'等'</span>, <span class="string">'u'</span>), (<span class="string">'技术性'</span>, <span class="string">'n'</span>), (<span class="string">'器件'</span>, <span class="string">'n'</span>), (<span class="string">'的'</span>, <span class="string">'u'</span>), (<span class="string">'安装'</span>, <span class="string">'vn'</span>), (<span class="string">'工作'</span>, <span class="string">'vn'</span>)]</span></span><br><span class="line"><span class="selector-tag">Sentiments</span>: 0<span class="selector-class">.015678817603646866</span></span><br><span class="line"><span class="selector-tag">Pinyin</span>: <span class="selector-attr">[<span class="string">'zhe'</span>, <span class="string">'ge'</span>, <span class="string">'ba'</span>, <span class="string">'shou'</span>, <span class="string">'gai'</span>, <span class="string">'huan'</span>, <span class="string">'liao'</span>, <span class="string">'，'</span>, <span class="string">'wo'</span>, <span class="string">'bu'</span>, <span class="string">'xi'</span>, <span class="string">'huan'</span>, <span class="string">'ri'</span>, <span class="string">'ben'</span>, <span class="string">'he'</span>, <span class="string">'fu'</span>, <span class="string">'，'</span>, <span class="string">'bie'</span>, <span class="string">'ba'</span>, <span class="string">'shou'</span>, <span class="string">'fang'</span>, <span class="string">'zai'</span>, <span class="string">'wo'</span>, <span class="string">'de'</span>, <span class="string">'jian'</span>, <span class="string">'bang'</span>, <span class="string">'shang'</span>, <span class="string">'，'</span>, <span class="string">'gong'</span>, <span class="string">'xin'</span>, <span class="string">'chu'</span>, <span class="string">'nv'</span>, <span class="string">'gan'</span>, <span class="string">'shi'</span>, <span class="string">'mei'</span>, <span class="string">'yue'</span>, <span class="string">'jing'</span>, <span class="string">'guo'</span>, <span class="string">'xia'</span>, <span class="string">'shu'</span>, <span class="string">'ke'</span>, <span class="string">'shi'</span>, <span class="string">'dou'</span>, <span class="string">'yao'</span>, <span class="string">'qin'</span>, <span class="string">'kou'</span>, <span class="string">'jiao'</span>, <span class="string">'dai'</span>, <span class="string">'24'</span>, <span class="string">'kou'</span>, <span class="string">'jiao'</span>, <span class="string">'huan'</span>, <span class="string">'ji'</span>, <span class="string">'deng'</span>, <span class="string">'ji'</span>, <span class="string">'shu'</span>, <span class="string">'xing'</span>, <span class="string">'qi'</span>, <span class="string">'jian'</span>, <span class="string">'de'</span>, <span class="string">'an'</span>, <span class="string">'zhuang'</span>, <span class="string">'gong'</span>, <span class="string">'zuo'</span>]</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="3-THULAC"><a href="#3-THULAC" class="headerlink" title="3. THULAC"></a>3. THULAC</h3>
                  <p>THULAC（THU Lexical Analyzer for Chinese）由清华大学自然语言处理与社会人文计算实验室研制推出的一套中文词法分析工具包，GitHub 链接：<a href="https://github.com/thunlp/THULAC-Python" target="_blank" rel="noopener">https://github.com/thunlp/THULAC-Python</a>，具有中文分词和词性标注功能。THULAC具有如下几个特点：</p>
                  <ul>
                    <li>能力强。利用集成的目前世界上规模最大的人工分词和词性标注中文语料库（约含5800万字）训练而成，模型标注能力强大。</li>
                    <li>准确率高。该工具包在标准数据集Chinese Treebank（CTB5）上分词的F1值可达97.3％，词性标注的F1值可达到92.9％，与该数据集上最好方法效果相当。</li>
                    <li>速度较快。同时进行分词和词性标注速度为300KB/s，每秒可处理约15万字。只进行分词速度可达到1.3MB/s。</li>
                  </ul>
                  <p>我们用一个实例看一下分词效果：</p>
                  <figure class="highlight go">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> thulac</span><br><span class="line"></span><br><span class="line"><span class="keyword">string</span> = <span class="string">'这个把手该换了，我不喜欢日本和服，别把手放在我的肩膀上，工信处女干事每月经过下属科室都要亲口交代24口交换机等技术性器件的安装工作'</span></span><br><span class="line">t = thulac.thulac()</span><br><span class="line">result = t.cut(<span class="keyword">string</span>)</span><br><span class="line"><span class="built_in">print</span>(result)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[[<span class="symbol">'这个</span>', <span class="symbol">'r</span>'], [<span class="symbol">'把手</span>', <span class="symbol">'n</span>'], [<span class="symbol">'该</span>', <span class="symbol">'v</span>'], [<span class="symbol">'换</span>', <span class="symbol">'v</span>'], [<span class="symbol">'了</span>', <span class="symbol">'u</span>'], [<span class="symbol">'，</span>', <span class="symbol">'w</span>'], [<span class="symbol">'我</span>', <span class="symbol">'r</span>'], [<span class="symbol">'不</span>', <span class="symbol">'d</span>'], [<span class="symbol">'喜欢</span>', <span class="symbol">'v</span>'], [<span class="symbol">'日本</span>', <span class="symbol">'ns</span>'], [<span class="symbol">'和服</span>', <span class="symbol">'n</span>'], [<span class="symbol">'，</span>', <span class="symbol">'w</span>'], [<span class="symbol">'别把手</span>', <span class="symbol">'n</span>'], [<span class="symbol">'放</span>', <span class="symbol">'v</span>'], [<span class="symbol">'在</span>', <span class="symbol">'p</span>'], [<span class="symbol">'我</span>', <span class="symbol">'r</span>'], [<span class="symbol">'的</span>', <span class="symbol">'u</span>'], [<span class="symbol">'肩膀</span>', <span class="symbol">'n</span>'], [<span class="symbol">'上</span>', <span class="symbol">'f</span>'], [<span class="symbol">'，</span>', <span class="symbol">'w</span>'], [<span class="symbol">'工信处</span>', <span class="symbol">'n</span>'], [<span class="symbol">'女</span>', <span class="symbol">'a</span>'], [<span class="symbol">'干事</span>', <span class="symbol">'n</span>'], [<span class="symbol">'每月</span>', <span class="symbol">'r</span>'], [<span class="symbol">'经过</span>', <span class="symbol">'p</span>'], [<span class="symbol">'下属</span>', <span class="symbol">'v</span>'], [<span class="symbol">'科室</span>', <span class="symbol">'n</span>'], [<span class="symbol">'都</span>', <span class="symbol">'d</span>'], [<span class="symbol">'要</span>', <span class="symbol">'v</span>'], [<span class="symbol">'亲口</span>', <span class="symbol">'d</span>'], [<span class="symbol">'交代</span>', <span class="symbol">'v</span>'], [<span class="symbol">'24</span>', <span class="symbol">'m</span>'], [<span class="symbol">'口</span>', <span class="symbol">'q</span>'], [<span class="symbol">'交换机</span>', <span class="symbol">'n</span>'], [<span class="symbol">'等</span>', <span class="symbol">'u</span>'], [<span class="symbol">'技术性</span>', <span class="symbol">'n</span>'], [<span class="symbol">'器件</span>', <span class="symbol">'n</span>'], [<span class="symbol">'的</span>', <span class="symbol">'u</span>'], [<span class="symbol">'安装</span>', <span class="symbol">'v</span>'], [<span class="symbol">'工作</span>', <span class="symbol">'v</span>']]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="4-NLPIR"><a href="#4-NLPIR" class="headerlink" title="4. NLPIR"></a>4. NLPIR</h3>
                  <p>NLPIR 分词系统，前身为2000年发布的 ICTCLAS 词法分析系统，GitHub 链接：<a href="https://github.com/NLPIR-team/NLPIR" target="_blank" rel="noopener">https://github.com/NLPIR-team/NLPIR</a>，是由北京理工大学张华平博士研发的中文分词系统，经过十余年的不断完善，拥有丰富的功能和强大的性能。NLPIR是一整套对原始文本集进行处理和加工的软件，提供了中间件处理效果的可视化展示，也可以作为小规模数据的处理加工工具。主要功能包括：中文分词，词性标注，命名实体识别，用户词典、新词发现与关键词提取等功能。另外对于分词功能，它有 Python 实现的版本，GitHub 链接：<a href="https://github.com/tsroten/pynlpir" target="_blank" rel="noopener">https://github.com/tsroten/pynlpir</a>。 使用方法如下：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import pynlpir</span><br><span class="line"></span><br><span class="line">pynlpir.<span class="built_in">open</span>()</span><br><span class="line"><span class="keyword">string</span> = <span class="string">'这个把手该换了，我不喜欢日本和服，别把手放在我的肩膀上，工信处女干事每月经过下属科室都要亲口交代24口交换机等技术性器件的安装工作'</span></span><br><span class="line"><span class="built_in">result</span> = pynlpir.<span class="keyword">segment</span>(<span class="keyword">string</span>)</span><br><span class="line">print(<span class="built_in">result</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[(<span class="symbol">'这个</span>', <span class="symbol">'pronoun</span>'), (<span class="symbol">'把</span>', <span class="symbol">'preposition</span>'), (<span class="symbol">'手</span>', <span class="symbol">'noun</span>'), (<span class="symbol">'该</span>', <span class="symbol">'pronoun</span>'), (<span class="symbol">'换</span>', <span class="symbol">'verb</span>'), (<span class="symbol">'了</span>', <span class="symbol">'modal</span> particle'), (<span class="symbol">'，</span>', <span class="symbol">'punctuation</span> mark'), (<span class="symbol">'我</span>', <span class="symbol">'pronoun</span>'), (<span class="symbol">'不</span>', <span class="symbol">'adverb</span>'), (<span class="symbol">'喜欢</span>', <span class="symbol">'verb</span>'), (<span class="symbol">'日本</span>', <span class="symbol">'noun</span>'), (<span class="symbol">'和</span>', <span class="symbol">'conjunction</span>'), (<span class="symbol">'服</span>', <span class="symbol">'verb</span>'), (<span class="symbol">'，</span>', <span class="symbol">'punctuation</span> mark'), (<span class="symbol">'别</span>', <span class="symbol">'adverb</span>'), (<span class="symbol">'把</span>', <span class="symbol">'preposition</span>'), (<span class="symbol">'手</span>', <span class="symbol">'noun</span>'), (<span class="symbol">'放</span>', <span class="symbol">'verb</span>'), (<span class="symbol">'在</span>', <span class="symbol">'preposition</span>'), (<span class="symbol">'我</span>', <span class="symbol">'pronoun</span>'), (<span class="symbol">'的</span>', <span class="symbol">'particle</span>'), (<span class="symbol">'肩膀</span>', <span class="symbol">'noun</span>'), (<span class="symbol">'上</span>', <span class="symbol">'noun</span> of locality'), (<span class="symbol">'，</span>', <span class="symbol">'punctuation</span> mark'), (<span class="symbol">'工</span>', <span class="symbol">'noun</span>'), (<span class="symbol">'信</span>', <span class="symbol">'noun</span>'), (<span class="symbol">'处女</span>', <span class="symbol">'noun</span>'), (<span class="symbol">'干事</span>', <span class="symbol">'noun</span>'), (<span class="symbol">'每月</span>', <span class="symbol">'pronoun</span>'), (<span class="symbol">'经过</span>', <span class="symbol">'preposition</span>'), (<span class="symbol">'下属</span>', <span class="symbol">'verb</span>'), (<span class="symbol">'科室</span>', <span class="symbol">'noun</span>'), (<span class="symbol">'都</span>', <span class="symbol">'adverb</span>'), (<span class="symbol">'要</span>', <span class="symbol">'verb</span>'), (<span class="symbol">'亲口</span>', <span class="symbol">'adverb</span>'), (<span class="symbol">'交代</span>', <span class="symbol">'verb</span>'), (<span class="symbol">'24</span>', <span class="symbol">'numeral</span>'), (<span class="symbol">'口</span>', <span class="symbol">'classifier</span>'), (<span class="symbol">'交换机</span>', <span class="symbol">'noun</span>'), (<span class="symbol">'等</span>', <span class="symbol">'particle</span>'), (<span class="symbol">'技术性</span>', <span class="symbol">'noun</span>'), (<span class="symbol">'器件</span>', <span class="symbol">'noun</span>'), (<span class="symbol">'的</span>', <span class="symbol">'particle</span>'), (<span class="symbol">'安装</span>', <span class="symbol">'verb</span>'), (<span class="symbol">'工作</span>', <span class="symbol">'verb</span>')]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里 把手 和 和服 也被分开了。</p>
                  <h3 id="5-NLTK"><a href="#5-NLTK" class="headerlink" title="5. NLTK"></a>5. NLTK</h3>
                  <p>NLTK，Natural Language Toolkit，是一个自然语言处理的包工具，各种多种 NLP 处理相关功能，GitHub 链接：<a href="https://github.com/nltk/nltk" target="_blank" rel="noopener">https://github.com/nltk/nltk</a>。 但是 NLTK 对于中文分词是不支持的，示例如下：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">from</span> <span class="variable">nltk</span> <span class="variable">import</span> <span class="variable">word_tokenize</span></span><br><span class="line"></span><br><span class="line"><span class="variable">string</span> = <span class="string">'这个把手该换了，我不喜欢日本和服，别把手放在我的肩膀上，工信处女干事每月经过下属科室都要亲口交代24口交换机等技术性器件的安装工作'</span></span><br><span class="line"><span class="variable"><span class="class">result</span></span> = <span class="function"><span class="title">word_tokenize</span>(<span class="variable">string</span>)</span></span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="variable"><span class="class">result</span></span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[<span class="symbol">'这个把手该换了，我不喜欢日本和服，别把手放在我的肩膀上，工信处女干事每月经过下属科室都要亲口交代24口交换机等技术性器件的安装工作</span>']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果要用中文分词的话，可以使用 FoolNLTK，它使用 Bi-LSTM 训练而成，包含分词、词性标注、实体识别等功能，同时支持自定义词典，可以训练自己的模型，可以进行批量处理。 使用方法如下：</p>
                  <figure class="highlight go">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> fool</span><br><span class="line"></span><br><span class="line"><span class="keyword">string</span> = <span class="string">'这个把手该换了，我不喜欢日本和服，别把手放在我的肩膀上，工信处女干事每月经过下属科室都要亲口交代24口交换机等技术性器件的安装工作'</span></span><br><span class="line">result = fool.cut(<span class="keyword">string</span>)</span><br><span class="line"><span class="built_in">print</span>(result)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[[<span class="symbol">'这个</span>', <span class="symbol">'把手</span>', <span class="symbol">'该</span>', <span class="symbol">'换</span>', <span class="symbol">'了</span>', <span class="symbol">'，</span>', <span class="symbol">'我</span>', <span class="symbol">'不</span>', <span class="symbol">'喜欢</span>', <span class="symbol">'日本</span>', <span class="symbol">'和服</span>', <span class="symbol">'，</span>', <span class="symbol">'别</span>', <span class="symbol">'把</span>', <span class="symbol">'手</span>', <span class="symbol">'放</span>', <span class="symbol">'在</span>', <span class="symbol">'我</span>', <span class="symbol">'的</span>', <span class="symbol">'肩膀</span>', <span class="symbol">'上</span>', <span class="symbol">'，</span>', <span class="symbol">'工信处</span>', <span class="symbol">'女</span>', <span class="symbol">'干事</span>', <span class="symbol">'每月</span>', <span class="symbol">'经过</span>', <span class="symbol">'下属</span>', <span class="symbol">'科室</span>', <span class="symbol">'都</span>', <span class="symbol">'要</span>', <span class="symbol">'亲</span>', <span class="symbol">'口</span>', <span class="symbol">'交代</span>', <span class="symbol">'24</span>', <span class="symbol">'口</span>', <span class="symbol">'交换机</span>', <span class="symbol">'等</span>', <span class="symbol">'技术性</span>', <span class="symbol">'器件</span>', <span class="symbol">'的</span>', <span class="symbol">'安装</span>', <span class="symbol">'工作</span>']]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到这个分词效果还是不错的。 另外还可以进行词性标注，实体识别：</p>
                  <figure class="highlight gauss">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">result = fool.pos_cut(<span class="keyword">string</span>)</span><br><span class="line"><span class="keyword">print</span>(result)</span><br><span class="line">_, ners = fool.analysis(<span class="keyword">string</span>)</span><br><span class="line"><span class="keyword">print</span>(ners)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[[(<span class="symbol">'这个</span>', <span class="symbol">'r</span>'), (<span class="symbol">'把手</span>', <span class="symbol">'n</span>'), (<span class="symbol">'该</span>', <span class="symbol">'r</span>'), (<span class="symbol">'换</span>', <span class="symbol">'v</span>'), (<span class="symbol">'了</span>', <span class="symbol">'y</span>'), (<span class="symbol">'，</span>', <span class="symbol">'wd</span>'), (<span class="symbol">'我</span>', <span class="symbol">'r</span>'), (<span class="symbol">'不</span>', <span class="symbol">'d</span>'), (<span class="symbol">'喜欢</span>', <span class="symbol">'vi</span>'), (<span class="symbol">'日本</span>', <span class="symbol">'ns</span>'), (<span class="symbol">'和服</span>', <span class="symbol">'n</span>'), (<span class="symbol">'，</span>', <span class="symbol">'wd</span>'), (<span class="symbol">'别</span>', <span class="symbol">'d</span>'), (<span class="symbol">'把</span>', <span class="symbol">'pba</span>'), (<span class="symbol">'手</span>', <span class="symbol">'n</span>'), (<span class="symbol">'放</span>', <span class="symbol">'v</span>'), (<span class="symbol">'在</span>', <span class="symbol">'p</span>'), (<span class="symbol">'我</span>', <span class="symbol">'r</span>'), (<span class="symbol">'的</span>', <span class="symbol">'ude</span>'), (<span class="symbol">'肩膀</span>', <span class="symbol">'n</span>'), (<span class="symbol">'上</span>', <span class="symbol">'f</span>'), (<span class="symbol">'，</span>', <span class="symbol">'wd</span>'), (<span class="symbol">'工信处</span>', <span class="symbol">'ns</span>'), (<span class="symbol">'女</span>', <span class="symbol">'b</span>'), (<span class="symbol">'干事</span>', <span class="symbol">'n</span>'), (<span class="symbol">'每月</span>', <span class="symbol">'r</span>'), (<span class="symbol">'经过</span>', <span class="symbol">'p</span>'), (<span class="symbol">'下属</span>', <span class="symbol">'v</span>'), (<span class="symbol">'科室</span>', <span class="symbol">'n</span>'), (<span class="symbol">'都</span>', <span class="symbol">'d</span>'), (<span class="symbol">'要</span>', <span class="symbol">'v</span>'), (<span class="symbol">'亲</span>', <span class="symbol">'a</span>'), (<span class="symbol">'口</span>', <span class="symbol">'n</span>'), (<span class="symbol">'交代</span>', <span class="symbol">'v</span>'), (<span class="symbol">'24</span>', <span class="symbol">'m</span>'), (<span class="symbol">'口</span>', <span class="symbol">'q</span>'), (<span class="symbol">'交换机</span>', <span class="symbol">'n</span>'), (<span class="symbol">'等</span>', <span class="symbol">'udeng</span>'), (<span class="symbol">'技术性</span>', <span class="symbol">'n</span>'), (<span class="symbol">'器件</span>', <span class="symbol">'n</span>'), (<span class="symbol">'的</span>', <span class="symbol">'ude</span>'), (<span class="symbol">'安装</span>', <span class="symbol">'n</span>'), (<span class="symbol">'工作</span>', <span class="symbol">'n</span>')]]</span><br><span class="line">[[(<span class="name">12</span>, <span class="number">15</span>, <span class="symbol">'location</span>', <span class="symbol">'日本</span>')]]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="6-LTP"><a href="#6-LTP" class="headerlink" title="6. LTP"></a>6. LTP</h3>
                  <p>语言技术平台（Language Technology Platform，LTP）是哈工大社会计算与信息检索研究中心历时十年开发的一整套中文语言处理系统。LTP制定了基于XML的语言处理结果表示，并在此基础上提供了一整套自底向上的丰富而且高效的中文语言处理模块（包括词法、句法、语义等6项中文处理核心技术），以及基于动态链接库（Dynamic Link Library, DLL）的应用程序接口、可视化工具，并且能够以网络服务（Web Service）的形式进行使用。 LTP 有 Python 版本，GitHub地址：<a href="https://github.com/HIT-SCIR/pyltp" target="_blank" rel="noopener">https://github.com/HIT-SCIR/pyltp</a>，另外运行的时候需要下载模型，模型还比较大，下载地址：<a href="http://ltp.ai/download.html" target="_blank" rel="noopener">http://ltp.ai/download.html</a>。 示例代码如下：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">from</span> <span class="variable">pyltp</span> <span class="variable">import</span> <span class="variable">Segmentor</span></span><br><span class="line"></span><br><span class="line"><span class="variable">string</span> = <span class="string">'这个把手该换了，我不喜欢日本和服，别把手放在我的肩膀上，工信处女干事每月经过下属科室都要亲口交代24口交换机等技术性器件的安装工作'</span></span><br><span class="line"><span class="variable">segmentor</span> = <span class="function"><span class="title">Segmentor</span>()</span></span><br><span class="line"><span class="variable">segmentor.load</span>(<span class="string">'./cws.model'</span>)</span><br><span class="line"><span class="variable"><span class="class">result</span></span> = <span class="function"><span class="title">list</span>(<span class="variable">segmentor.segment</span>(<span class="variable">string</span>))</span></span><br><span class="line"><span class="variable">segmentor.release</span>()</span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="variable"><span class="class">result</span></span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">41</span> 这个<span class="regexp">/把手/</span>该<span class="regexp">/换/</span>了<span class="regexp">/，/</span>我<span class="regexp">/不/</span>喜欢<span class="regexp">/日本/</span>和服<span class="regexp">/，/</span>别<span class="regexp">/把/</span>手<span class="regexp">/放在/</span>我<span class="regexp">/的/</span>肩膀<span class="regexp">/上/</span>，<span class="regexp">/工信/</span>处女<span class="regexp">/干事/</span>每月<span class="regexp">/经过/</span>下属<span class="regexp">/科室/</span>都<span class="regexp">/要/</span>亲口<span class="regexp">/交代/</span><span class="number">24</span><span class="regexp">/口/</span>交换机<span class="regexp">/等/</span>技术性<span class="regexp">/器件/</span>的<span class="regexp">/安装/</span>工作</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以发现 工信处、女干事 没有正确分开。 以上便是一些分词库的基本使用，个人比较推荐的有 jieba、THULAC、FoolNLTK。</p>
                  <h2 id="参考来源"><a href="#参考来源" class="headerlink" title="参考来源"></a>参考来源</h2>
                  <ul>
                    <li><a href="http://m635674608.iteye.com/blog/2298833" target="_blank" rel="noopener">http://m635674608.iteye.com/blog/2298833</a></li>
                    <li><a href="http://blog.csdn.net/flysky1991/article/details/73948971" target="_blank" rel="noopener">http://blog.csdn.net/flysky1991/article/details/73948971</a></li>
                  </ul>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-03-15 21:32:47" itemprop="dateCreated datePublished" datetime="2018-03-15T21:32:47+08:00">2018-03-15</time>
                </span>
                <span id="/5844.html" class="post-meta-item leancloud_visitors" data-flag-title="中文分词原理及工具" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>11k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>10 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5822.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5822.html" class="post-title-link" itemprop="url">深度学习 GPU环境 Ubuntu 16.04 + Nvidia GTX 1080 + Python 3.6 + CUDA 9.0 + cuDNN 7.1 + TensorFlow 1.6 环境配置</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>本节详细说明一下深度学习环境配置，Ubuntu 16.04 + Nvidia GTX 1080 + Python 3.6 + CUDA 9.0 + cuDNN 7.1 + TensorFlow 1.6。</p>
                  <h2 id="Python-3-6"><a href="#Python-3-6" class="headerlink" title="Python 3.6"></a>Python 3.6</h2>
                  <p>首先安装 Python 3.6，这里使用 Anaconda 3 来安装，下载地址：<a href="https://www.anaconda.com/download/#linux" target="_blank" rel="noopener">https://www.anaconda.com/download/#linux</a>，点击 Download 按钮下载即可，这里下载的是 Anaconda 3-5.1 版本，如果下载速度过慢可以选择使用清华镜像：<a href="https://mirrors.tuna.tsinghua.edu.cn/anaconda/archive/" target="_blank" rel="noopener">https://mirrors.tuna.tsinghua.edu.cn/anaconda/archive/</a>。 下载下来之后目录下会出现一个 Anaconda3-5.1.0-Linux-x86_64.sh 文件，然后直接执行即可安装：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">bash</span> <span class="selector-tag">Anaconda3-5</span><span class="selector-class">.1</span><span class="selector-class">.0-Linux-x86_64</span><span class="selector-class">.sh</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>执行完毕之后按照默认设置走下来即可完成安装。 这里默认它会安装到用户目录下，如果想全局安装，可以在这一步输入你要安装的地址：</p>
                  <figure class="highlight groovy">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Anaconda3 will now be installed into <span class="keyword">this</span> <span class="string">location:</span></span><br><span class="line"><span class="regexp">/home/</span>cqc/anaconda3</span><br><span class="line"></span><br><span class="line">  - Press ENTER to confirm the location</span><br><span class="line">  - Press CTRL-C to abort the installation</span><br><span class="line">  - Or specify a different location below</span><br><span class="line"></span><br><span class="line">[<span class="regexp">/home/</span>cqc<span class="regexp">/anaconda3] &gt;&gt;&gt; /</span>usr<span class="regexp">/local/</span>anaconda3</span><br><span class="line">PREFIX=<span class="regexp">/usr/</span>local/anaconda3</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我指定了将其安装到 /usr/local/anaconda3 目录下，全局安装，所有用户共享，当然如果只想本用户使用的话使用默认配置即可。 安装完成之后添加 python3 和 pip3 的软链接：</p>
                  <figure class="highlight dts">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo ln -s <span class="meta-keyword">/usr/</span>local<span class="meta-keyword">/anaconda3/</span>bin/python3 <span class="meta-keyword">/usr/</span>local<span class="meta-keyword">/sbin/</span>python3</span><br><span class="line">sudo ln -s <span class="meta-keyword">/usr/</span>local<span class="meta-keyword">/anaconda3/</span>bin/pip <span class="meta-keyword">/usr/</span>local<span class="meta-keyword">/sbin/</span>pip3</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里是将软连接其添加到 /usr/local/sbin 目录下了，它默认会存在于环境变量中，因此可以直接调用。 当然也可以选择把 /usr/local/anaconda3/bin 目录添加到环境变量中，可以修改 ~/.bashrc 文件，添加如下内容：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="builtin-name">export</span> <span class="attribute">PATH</span>=/usr/local/anaconda3/bin$&#123;PATH:+:<span class="variable">$&#123;PATH&#125;</span>&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后执行：</p>
                  <figure class="highlight bash">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">source</span> ~/.bashrc</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>即可生效，下次登录时也会默认执行 ~/.bashrc 文件，也会生效。 接下来我们验证下 python3、pip3 命令是否都来自 Anaconda，命令如下：</p>
                  <figure class="highlight crystal">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 -V</span><br><span class="line">pip <span class="number">9.0</span>.<span class="number">1</span> from /usr/local/anaconda3/<span class="class"><span class="keyword">lib</span>/<span class="title">python3</span>.6/<span class="title">site</span>-<span class="title">packages</span> (<span class="title">python</span> 3.6)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">which python3</span><br><span class="line">/usr/local/anaconda3/bin/python3</span><br><span class="line">python3</span><br><span class="line">Python 3.6.4 |Anaconda, Inc.| (default, Jan 16 2018, 18:10:19) </span><br><span class="line">[GCC 7.2.0] on linux</span><br><span class="line">Type <span class="string">"help"</span>, <span class="string">"copyright"</span>, <span class="string">"credits"</span> <span class="keyword">or</span> <span class="string">"license"</span> <span class="keyword">for</span> more information.</span><br><span class="line">&gt;&gt;&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果输入 pip3 和 python3 命令能出现如上类似结果，路径都在 /usr/local/anaconda3，就证明 Python 3 安装成功了。</p>
                  <h2 id="安装驱动"><a href="#安装驱动" class="headerlink" title="安装驱动"></a>安装驱动</h2>
                  <p>首先查看一下自己的电脑需要怎样的驱动，我们可以先到 <a href="http://www.nvidia.com/Download/index.aspx" target="_blank" rel="noopener">http://www.nvidia.com/Download/index.aspx</a> 查询下我们需要的是怎样的驱动，这里我的显卡是 GTX 1080，所以以此为例说明，勾选好对应的配置： <img src="https://germey.gitbooks.io/ai/content/assets/2018-03-11-18-46-41.jpg" alt=""> 点击 Search，可以看到查询结果如下所示：</p>
                  <figure class="highlight yaml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">Version:</span>    <span class="number">390.25</span></span><br><span class="line"><span class="attr">Release Date:</span>    <span class="number">2018.1</span><span class="number">.29</span></span><br><span class="line"><span class="attr">Operating System:</span>    <span class="string">Linux</span> <span class="number">64</span><span class="string">-bit</span></span><br><span class="line"><span class="attr">Language:</span>    <span class="string">English</span> <span class="string">(US)</span></span><br><span class="line"><span class="attr">File Size:</span>    <span class="number">77.48</span> <span class="string">MB</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里说明我们需要的版本是 390.25。 接下来如果我们之前安装了驱动的话，可以重新安装一下，如果当前已经安装好了就不必了。 如果要重装，需要首先卸载掉之前的显卡驱动：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo apt-<span class="builtin-name">get</span> <span class="builtin-name">remove</span> –purge nvidia*</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行之后 NVIDIA 的一些驱动就被卸载了。 这时候 nvidia-smi 等命令已经不能用了，这就证明显卡驱动已经被卸载了。 然后接下来添加一个 PPA 源，命令如下：</p>
                  <figure class="highlight smali">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo<span class="built_in"> add-apt-repository </span>ppa:graphics-drivers/ppa</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后更新一下：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo apt-<span class="keyword">get</span> <span class="keyword">update</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>随后重新安装显卡驱动：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo apt-<span class="builtin-name">get</span> install nvidia-390</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>注意这里的 390 就是刚才我们查询出来的版本，以实际查询出来的版本为准。</p>
                  <h2 id="CUDA-9-0"><a href="#CUDA-9-0" class="headerlink" title="CUDA 9.0"></a>CUDA 9.0</h2>
                  <p>如果存在之前的旧版本，可以选择先卸载，以免和新的 CUDA 版本产生冲突，在 /usr/local/cuda/bin 目录下有一个 uninstallcuda*.pl 文件，可以直接运行卸载，命令如下：</p>
                  <figure class="highlight jboss-cli">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo <span class="string">./uninstall_cuda_</span>*<span class="string">.pl</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样即可将 CUDA 全部卸载。 接下来我们再下载 CUDA 9.0，注意 TensorFlow 1.5 和 1.6 版本依然只是兼容 CUDA 9.0，没有兼容 CUDA 9.1，所以不要下载 9.1，CUDA 9.0 的下载地址是：<a href="https://developer.nvidia.com/cuda-90-download-archive" target="_blank" rel="noopener">https://developer.nvidia.com/cuda-90-download-archive</a>，然后依次勾选好系统的版本，如图所示： <img src="https://germey.gitbooks.io/ai/content/assets/2018-03-11-23-37-55.jpg" alt=""> 这里我们选择 Linux-x86_64-Ubuntu-16.04-runfile 的配置，然后点击 Base Installer 部分的 Download 按钮，下载 CUDA 9.0 安装包。 对应的下载命令是：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">wget https:<span class="regexp">//</span>developer.nvidia.com<span class="regexp">/compute/</span>cuda<span class="regexp">/9.0/</span>Prod<span class="regexp">/local_installers/</span>cuda_9.<span class="number">0.176</span>_384.<span class="number">81</span>_linux-run</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>执行此命令，等待下载完成即可。 接下来执行安装，运行如下命令：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo bash cuda_9<span class="number">.0</span><span class="number">.176</span>_384<span class="number">.81</span>_linux-run</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>安装过程需要输入一些确认选项，过程如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Description</span><br><span class="line"></span><br><span class="line">The NVIDIA CUDA Toolkit provides command-line <span class="keyword">and</span> graphical</span><br><span class="line">tools <span class="keyword">for</span> building, debugging <span class="keyword">and</span> optimizing the performance</span><br><span class="line"><span class="keyword">Do</span> you accept the previously read EULA?</span><br><span class="line">accept/decline/quit: accept</span><br><span class="line"></span><br><span class="line">Install NVIDIA Accelerated Graphics Driver <span class="keyword">for</span> Linux-x86_64 384.81?</span><br><span class="line">(y)es/(n)o/(q)uit: n</span><br><span class="line"></span><br><span class="line">Install the CUDA 9.0 Toolkit?</span><br><span class="line">(y)es/(n)o/(q)uit: y</span><br><span class="line"></span><br><span class="line">Enter Toolkit Location</span><br><span class="line"> [<span class="built_in"> default </span>is /usr/local/cuda-9.0 ]: </span><br><span class="line"></span><br><span class="line"><span class="keyword">Do</span> you want <span class="keyword">to</span> install a symbolic link at /usr/local/cuda?</span><br><span class="line">(y)es/(n)o/(q)uit: y</span><br><span class="line"></span><br><span class="line">Install the CUDA 9.0 Samples?</span><br><span class="line">(y)es/(n)o/(q)uit: y</span><br><span class="line"></span><br><span class="line">Enter CUDA Samples Location</span><br><span class="line"> [<span class="built_in"> default </span>is /home/cqc ]: </span><br><span class="line"></span><br><span class="line">Installing the CUDA Toolkit <span class="keyword">in</span> /usr/local/cuda-9.0 <span class="built_in">..</span>.</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>最后如果出现这样的提示，就证明 CUDA 安装好了：</p>
                  <figure class="highlight">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">Driver</span>:   Not Selected</span><br><span class="line"><span class="attribute">Toolkit</span>:  Installed in /usr/local/cuda-9.0</span><br><span class="line"><span class="attribute">Samples</span>:  Installed in /home/cqc, but missing recommended libraries</span><br><span class="line"></span><br><span class="line">Please make sure that</span><br><span class="line"> -   PATH includes /usr/local/cuda-9.0/bin</span><br><span class="line"> -   LD_LIBRARY_PATH includes /usr/local/cuda-9.0/lib64, or, add /usr/local/cuda-9.0/lib64 to /etc/ld.so.conf and run ldconfig as root</span><br><span class="line"></span><br><span class="line">To uninstall the CUDA Toolkit, run the uninstall script in /usr/local/cuda-9.0/bin</span><br><span class="line"></span><br><span class="line">Please see CUDA_Installation_Guide_Linux.pdf in /usr/local/cuda-9.0/doc/pdf for detailed information on setting up CUDA.</span><br><span class="line"></span><br><span class="line">***WARNING: Incomplete installation! This installation did not install the CUDA Driver. A driver of version at least 384.00 is required for CUDA 9.0 functionality to work.</span><br><span class="line">To install the driver using this installer, run the following command, replacing &lt;CudaInstaller&gt; with the name of this run file:</span><br><span class="line">    sudo &lt;CudaInstaller&gt;.run -silent -driver</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后我们需要配置一下环境变量，更改 ~/.bashrc 文件，添加如下几行：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="builtin-name">export</span> <span class="attribute">PATH</span>=/usr/local/cuda/bin$&#123;PATH:+:<span class="variable">$&#123;PATH&#125;</span>&#125;</span><br><span class="line"><span class="builtin-name">export</span> <span class="attribute">LD_LIBRARY_PATH</span>=/usr/local/cuda/lib64$&#123;LD_LIBRARY_PATH:+:<span class="variable">$&#123;LD_LIBRARY_PATH&#125;</span>&#125;</span><br><span class="line"><span class="builtin-name">export</span> <span class="attribute">CUDA_HOME</span>=/usr/local/cuda</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>修改完毕之后执行一下使其生效：</p>
                  <figure class="highlight bash">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">source</span> ~/.bashrc</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这时我们输出 CUDA_HOME、LD_LIBRARY_PATH 就可以看到对应的输出了：</p>
                  <figure class="highlight bash">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">echo</span> <span class="variable">$CUDA_HOME</span></span><br><span class="line">/usr/<span class="built_in">local</span>/cuda</span><br><span class="line"><span class="built_in">echo</span> <span class="variable">$LD_LIBRARY_PATH</span></span><br><span class="line">/usr/<span class="built_in">local</span>/cuda/lib64</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就代表环境变量生效了，CUDA 安装完成。</p>
                  <h2 id="cuDNN-7-1"><a href="#cuDNN-7-1" class="headerlink" title="cuDNN 7.1"></a>cuDNN 7.1</h2>
                  <p>cuDNN 的全称是 The NVIDIA CUDA® Deep Neural Network library，是专门用来对深度学习加速的库，它支持 Caffe2, MATLAB, Microsoft Cognitive Toolkit, TensorFlow, Theano 及 PyTorch 等深度学习的加速优化，目前最新版本是 cuDNN 7.1，接下来我们来看下它的安装方式。 下载链接：<a href="https://developer.nvidia.com/rdp/cudnn-download" target="_blank" rel="noopener">https://developer.nvidia.com/rdp/cudnn-download</a>，需要注册之后才能打开，这里我们选择 cuDNN v7.1.1 (Feb 28, 2018), for CUDA 9.0，然后选择 cuDNN v7.1.1 Library for Linux，如图所示： <img src="https://germey.gitbooks.io/ai/content/assets/2018-03-11-23-49-17.jpg" alt=""> 下载下来之后解压安装即可：</p>
                  <figure class="highlight dts">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">tar -zxvf cudnn<span class="number">-9.0</span>-linux-x64-v7<span class="number">.1</span>.tgz</span><br><span class="line">sudo cp cuda<span class="meta-keyword">/include/</span>cudnn.h <span class="meta-keyword">/usr/</span>local<span class="meta-keyword">/cuda/</span>include/</span><br><span class="line">sudo cp cuda<span class="meta-keyword">/lib64/</span>libcudnn* <span class="meta-keyword">/usr/</span>local<span class="meta-keyword">/cuda/</span>lib64/ -d</span><br><span class="line">sudo chmod a+r <span class="meta-keyword">/usr/</span>local<span class="meta-keyword">/cuda/</span>include/cudnn.h</span><br><span class="line">sudo chmod a+r <span class="meta-keyword">/usr/</span>local<span class="meta-keyword">/cuda/</span>lib64/libcudnn*</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>执行完如上命令之后，cuDNN 就安装好了，这时我们可以发现在 /usr/local/cuda/include 目录下就多了 cudnn.h 头文件。</p>
                  <h2 id="TensorFlow-1-6"><a href="#TensorFlow-1-6" class="headerlink" title="TensorFlow 1.6"></a>TensorFlow 1.6</h2>
                  <p>到现在为止 Python 3.6、CUDA 9.0 和 cuDNN 7.1 就已经安装好了，而且环境变量也配置好了，接下来我们直接安装 TensorFlow 1.6 即可，TensorFlow 1.6 版本针对 CUDA 9 和 cuDNN 7 做了优化，可以预构建二进制文件。 这里需要安装的是 TensorFlow 的 GPU 版本，命令如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 install tensorflow-gpu==<span class="number">1.6</span><span class="number">.0</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>安装完成之后验证一下：</p>
                  <figure class="highlight elm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> tensorflow</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果没有报错，那就证明全部环境配置都成功了。 以上便是 Ubuntu 16.04 + Nvidia GTX 1080 + Python 3.6 + CUDA 9.0 + cuDNN 7.1 + TensorFlow 1.6 完整环境配置过程。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-03-12 15:45:41" itemprop="dateCreated datePublished" datetime="2018-03-12T15:45:41+08:00">2018-03-12</time>
                </span>
                <span id="/5822.html" class="post-meta-item leancloud_visitors" data-flag-title="深度学习 GPU环境 Ubuntu 16.04 + Nvidia GTX 1080 + Python 3.6 + CUDA 9.0 + cuDNN 7.1 + TensorFlow 1.6 环境配置" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>5.5k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>5 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5806.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Linux <i class="label-arrow"></i>
                  </a>
                  <a href="/5806.html" class="post-title-link" itemprop="url">Linux常用命令总结</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="系统"><a href="#系统" class="headerlink" title="系统"></a>系统</h2>
                  <p>查看系统版本：</p>
                  <figure class="highlight stata">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">cat</span> /proc/<span class="keyword">version</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>实例结果：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Linux version <span class="number">4.4</span><span class="number">.0</span><span class="number">-112</span>-generic (<span class="symbol">buildd@</span>lgw01-amd64<span class="number">-010</span>) (gcc version <span class="number">5.4</span><span class="number">.0</span> <span class="number">20160609</span> (Ubuntu <span class="number">5.4</span><span class="number">.0</span><span class="number">-6</span>ubuntu1~<span class="number">16.04</span><span class="number">.5</span>) ) #<span class="number">135</span>-Ubuntu SMP Fri Jan <span class="number">19</span> <span class="number">11</span>:<span class="number">48</span>:<span class="number">36</span> UTC <span class="number">2018</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="CPU"><a href="#CPU" class="headerlink" title="CPU"></a>CPU</h2>
                  <p>查看CPU信息：</p>
                  <figure class="highlight armasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">cat</span> /<span class="meta">proc</span>/cpuinfo</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>示例结果：</p>
                  <figure class="highlight yaml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">processor       :</span> <span class="number">0</span></span><br><span class="line"><span class="attr">vendor_id       :</span> <span class="string">GenuineIntel</span></span><br><span class="line"><span class="attr">cpu family      :</span> <span class="number">6</span></span><br><span class="line"><span class="attr">model           :</span> <span class="number">63</span></span><br><span class="line"><span class="attr">model name      :</span> <span class="string">Intel(R)</span> <span class="string">Xeon(R)</span> <span class="string">CPU</span> <span class="string">E5-2673</span> <span class="string">v3</span> <span class="string">@</span> <span class="number">2.</span><span class="string">40GHz</span></span><br><span class="line"><span class="attr">stepping        :</span> <span class="number">2</span></span><br><span class="line"><span class="attr">microcode       :</span> <span class="number">0x3a</span></span><br><span class="line"><span class="attr">cpu MHz         :</span> <span class="number">1200.000</span></span><br><span class="line"><span class="attr">cache size      :</span> <span class="number">30720</span> <span class="string">KB</span></span><br><span class="line"><span class="attr">physical id     :</span> <span class="number">0</span></span><br><span class="line"><span class="attr">siblings        :</span> <span class="number">24</span></span><br><span class="line"><span class="attr">core id         :</span> <span class="number">0</span></span><br><span class="line"><span class="attr">cpu cores       :</span> <span class="number">12</span></span><br><span class="line"><span class="attr">apicid          :</span> <span class="number">0</span></span><br><span class="line"><span class="attr">initial apicid  :</span> <span class="number">0</span></span><br><span class="line"><span class="attr">fpu             :</span> <span class="literal">yes</span></span><br><span class="line"><span class="attr">fpu_exception   :</span> <span class="literal">yes</span></span><br><span class="line"><span class="attr">cpuid level     :</span> <span class="number">15</span></span><br><span class="line"><span class="attr">wp              :</span> <span class="literal">yes</span></span><br><span class="line"><span class="attr">flags           :</span> <span class="string">fpu</span> <span class="string">vme</span> <span class="string">de</span> <span class="string">pse</span> <span class="string">tsc</span> <span class="string">msr</span> <span class="string">pae</span> <span class="string">mce</span> <span class="string">cx8</span> <span class="string">apic</span> <span class="string">sep</span> <span class="string">mtrr</span> <span class="string">pge</span> <span class="string">mca</span> <span class="string">cmov</span> <span class="string">pat</span> <span class="string">pse36</span> <span class="string">clflush</span> <span class="string">dts</span> <span class="string">acpi</span> <span class="string">mmx</span> <span class="string">fxsr</span> <span class="string">sse</span> <span class="string">sse2</span> <span class="string">ss</span> <span class="string">ht</span> <span class="string">tm</span> <span class="string">pbe</span> <span class="string">syscall</span> <span class="string">nx</span> <span class="string">pdpe1gb</span> <span class="string">rdtscp</span> <span class="string">lm</span> <span class="string">constant_tsc</span> <span class="string">arch_perfmon</span> <span class="string">pebs</span> <span class="string">bts</span> <span class="string">rep_good</span> <span class="string">nopl</span> <span class="string">xtopology</span> <span class="string">nonstop_tsc</span> <span class="string">aperfmperf</span> <span class="string">eagerfpu</span> <span class="string">pni</span> <span class="string">pclmulqdq</span> <span class="string">dtes64</span> <span class="string">monitor</span> <span class="string">ds_cpl</span> <span class="string">vmx</span> <span class="string">smx</span> <span class="string">est</span> <span class="string">tm2</span> <span class="string">ssse3</span> <span class="string">sdbg</span> <span class="string">fma</span> <span class="string">cx16</span> <span class="string">xtpr</span> <span class="string">pdcm</span> <span class="string">pcid</span> <span class="string">dca</span> <span class="string">sse4_1</span> <span class="string">sse4_2</span> <span class="string">x2apic</span> <span class="string">movbe</span> <span class="string">popcnt</span> <span class="string">tsc_deadline_timer</span> <span class="string">aes</span> <span class="string">xsave</span> <span class="string">avx</span> <span class="string">f16c</span> <span class="string">rdrand</span> <span class="string">lahf_lm</span> <span class="string">abm</span> <span class="string">epb</span> <span class="string">invpcid_single</span> <span class="string">kaiser</span> <span class="string">tpr_shadow</span> <span class="string">vnmi</span> <span class="string">flexpriority</span> <span class="string">ept</span> <span class="string">vpid</span> <span class="string">fsgsbase</span> <span class="string">tsc_adjust</span> <span class="string">bmi1</span> <span class="string">avx2</span> <span class="string">smep</span> <span class="string">bmi2</span> <span class="string">erms</span> <span class="string">invpcid</span> <span class="string">cqm</span> <span class="string">xsaveopt</span> <span class="string">cqm_llc</span> <span class="string">cqm_occup_llc</span> <span class="string">dtherm</span> <span class="string">ida</span> <span class="string">arat</span> <span class="string">pln</span> <span class="string">pts</span></span><br><span class="line"><span class="attr">bugs            :</span></span><br><span class="line"><span class="attr">bogomips        :</span> <span class="number">4800.24</span></span><br><span class="line"><span class="attr">clflush size    :</span> <span class="number">64</span></span><br><span class="line"><span class="attr">cache_alignment :</span> <span class="number">64</span></span><br><span class="line"><span class="attr">address sizes   :</span> <span class="number">46</span> <span class="string">bits</span> <span class="string">physical,</span> <span class="number">48</span> <span class="string">bits</span> <span class="string">virtual</span></span><br><span class="line"><span class="attr">power management:</span></span><br><span class="line"></span><br><span class="line"><span class="attr">processor       :</span> <span class="number">1</span></span><br><span class="line"><span class="attr">vendor_id       :</span> <span class="string">GenuineIntel</span></span><br><span class="line"><span class="attr">cpu family      :</span> <span class="number">6</span></span><br><span class="line"><span class="attr">model           :</span> <span class="number">63</span></span><br><span class="line"><span class="attr">model name      :</span> <span class="string">Intel(R)</span> <span class="string">Xeon(R)</span> <span class="string">CPU</span> <span class="string">E5-2673</span> <span class="string">v3</span> <span class="string">@</span> <span class="number">2.</span><span class="string">40GHz</span></span><br><span class="line"><span class="attr">stepping        :</span> <span class="number">2</span></span><br><span class="line"><span class="attr">microcode       :</span> <span class="number">0x3a</span></span><br><span class="line"><span class="attr">cpu MHz         :</span> <span class="number">1207.687</span></span><br><span class="line"><span class="attr">cache size      :</span> <span class="number">30720</span> <span class="string">KB</span></span><br><span class="line"><span class="attr">physical id     :</span> <span class="number">0</span></span><br><span class="line"><span class="attr">siblings        :</span> <span class="number">24</span></span><br><span class="line"><span class="attr">core id         :</span> <span class="number">1</span></span><br><span class="line"><span class="attr">cpu cores       :</span> <span class="number">12</span></span><br><span class="line"><span class="attr">apicid          :</span> <span class="number">2</span></span><br><span class="line"><span class="attr">initial apicid  :</span> <span class="number">2</span></span><br><span class="line"><span class="attr">fpu             :</span> <span class="literal">yes</span></span><br><span class="line"><span class="attr">fpu_exception   :</span> <span class="literal">yes</span></span><br><span class="line"><span class="attr">cpuid level     :</span> <span class="number">15</span></span><br><span class="line"><span class="attr">wp              :</span> <span class="literal">yes</span></span><br><span class="line"><span class="attr">flags           :</span> <span class="string">fpu</span> <span class="string">vme</span> <span class="string">de</span> <span class="string">pse</span> <span class="string">tsc</span> <span class="string">msr</span> <span class="string">pae</span> <span class="string">mce</span> <span class="string">cx8</span> <span class="string">apic</span> <span class="string">sep</span> <span class="string">mtrr</span> <span class="string">pge</span> <span class="string">mca</span> <span class="string">cmov</span> <span class="string">pat</span> <span class="string">pse36</span> <span class="string">clflush</span> <span class="string">dts</span> <span class="string">acpi</span> <span class="string">mmx</span> <span class="string">fxsr</span> <span class="string">sse</span> <span class="string">sse2</span> <span class="string">ss</span> <span class="string">ht</span> <span class="string">tm</span> <span class="string">pbe</span> <span class="string">syscall</span> <span class="string">nx</span> <span class="string">pdpe1gb</span> <span class="string">rdtscp</span> <span class="string">lm</span> <span class="string">constant_tsc</span> <span class="string">arch_perfmon</span> <span class="string">pebs</span> <span class="string">bts</span> <span class="string">rep_good</span> <span class="string">nopl</span> <span class="string">xtopology</span> <span class="string">nonstop_tsc</span> <span class="string">aperfmperf</span> <span class="string">eagerfpu</span> <span class="string">pni</span> <span class="string">pclmulqdq</span> <span class="string">dtes64</span> <span class="string">monitor</span> <span class="string">ds_cpl</span> <span class="string">vmx</span> <span class="string">smx</span> <span class="string">est</span> <span class="string">tm2</span> <span class="string">ssse3</span> <span class="string">sdbg</span> <span class="string">fma</span> <span class="string">cx16</span> <span class="string">xtpr</span> <span class="string">pdcm</span> <span class="string">pcid</span> <span class="string">dca</span> <span class="string">sse4_1</span> <span class="string">sse4_2</span> <span class="string">x2apic</span> <span class="string">movbe</span> <span class="string">popcnt</span> <span class="string">tsc_deadline_timer</span> <span class="string">aes</span> <span class="string">xsave</span> <span class="string">avx</span> <span class="string">f16c</span> <span class="string">rdrand</span> <span class="string">lahf_lm</span> <span class="string">abm</span> <span class="string">epb</span> <span class="string">invpcid_single</span> <span class="string">kaiser</span> <span class="string">tpr_shadow</span> <span class="string">vnmi</span> <span class="string">flexpriority</span> <span class="string">ept</span> <span class="string">vpid</span> <span class="string">fsgsbase</span> <span class="string">tsc_adjust</span> <span class="string">bmi1</span> <span class="string">avx2</span> <span class="string">smep</span> <span class="string">bmi2</span> <span class="string">erms</span> <span class="string">invpcid</span> <span class="string">cqm</span> <span class="string">xsaveopt</span> <span class="string">cqm_llc</span> <span class="string">cqm_occup_llc</span> <span class="string">dtherm</span> <span class="string">ida</span> <span class="string">arat</span> <span class="string">pln</span> <span class="string">pts</span></span><br><span class="line"><span class="attr">bugs            :</span></span><br><span class="line"><span class="attr">bogomips        :</span> <span class="number">4800.24</span></span><br><span class="line"><span class="attr">clflush size    :</span> <span class="number">64</span></span><br><span class="line"><span class="attr">cache_alignment :</span> <span class="number">64</span></span><br><span class="line"><span class="attr">address sizes   :</span> <span class="number">46</span> <span class="string">bits</span> <span class="string">physical,</span> <span class="number">48</span> <span class="string">bits</span> <span class="string">virtual</span></span><br><span class="line"><span class="attr">power management:</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="内存"><a href="#内存" class="headerlink" title="内存"></a>内存</h2>
                  <p>查看内存及用量（M为单位）：</p>
                  <figure class="highlight cpp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">free</span> -m</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>示例结果：</p>
                  <figure class="highlight yaml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">              <span class="string">total</span>        <span class="string">used</span>        <span class="string">free</span>      <span class="string">shared</span>  <span class="string">buff/cache</span>   <span class="string">available</span></span><br><span class="line"><span class="attr">Mem:</span>         <span class="number">128806</span>        <span class="number">7115</span>      <span class="number">115910</span>          <span class="number">77</span>        <span class="number">5780</span>      <span class="number">120877</span></span><br><span class="line"><span class="attr">Swap:</span>             <span class="number">0</span>           <span class="number">0</span>           <span class="number">0</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>查看内存及用量（G为单位）：</p>
                  <figure class="highlight cpp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">free</span> -g</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>示例结果：</p>
                  <figure class="highlight yaml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">              <span class="string">total</span>        <span class="string">used</span>        <span class="string">free</span>      <span class="string">shared</span>  <span class="string">buff/cache</span>   <span class="string">available</span></span><br><span class="line"><span class="attr">Mem:</span>            <span class="number">125</span>           <span class="number">6</span>         <span class="number">113</span>           <span class="number">0</span>           <span class="number">5</span>         <span class="number">118</span></span><br><span class="line"><span class="attr">Swap:</span>             <span class="number">0</span>           <span class="number">0</span>           <span class="number">0</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="硬盘用量"><a href="#硬盘用量" class="headerlink" title="硬盘用量"></a>硬盘用量</h2>
                  <p>查看各分区使用情况：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">df -h</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>示例结果：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Filesystem      Size  Used Avail Use% Mounted on</span><br><span class="line">udev             <span class="number">63</span>G     <span class="number">0</span>   <span class="number">63</span>G   <span class="number">0</span>% /dev</span><br><span class="line">tmpfs            <span class="number">13</span>G   <span class="number">66</span>M   <span class="number">13</span>G   <span class="number">1</span>% /run</span><br><span class="line">/dev/sda1       <span class="number">224</span>G  <span class="number">101</span>G  <span class="number">113</span>G  <span class="number">48</span>% /</span><br><span class="line">tmpfs            <span class="number">63</span>G  <span class="number">352</span>K   <span class="number">63</span>G   <span class="number">1</span>% /dev/shm</span><br><span class="line">tmpfs           <span class="number">5.0</span>M  <span class="number">4.0</span>K  <span class="number">5.0</span>M   <span class="number">1</span>% /run/lock</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>查看当前目录文件及文件夹大小：</p>
                  <figure class="highlight mipsasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">du -<span class="keyword">sh </span>*</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>示例结果：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">6.9</span>G    anaconda3</span><br><span class="line"><span class="number">213</span>M    cuda_samples</span><br><span class="line"><span class="number">7.9</span>M    Desktop</span><br><span class="line"><span class="number">8.0</span>K    Documents</span><br><span class="line"><span class="number">7.1</span>G    Downloads</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="GPU"><a href="#GPU" class="headerlink" title="GPU"></a>GPU</h2>
                  <p>查看 GPU 使用情况：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">nvidia-smi</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>示例结果：</p>
                  <figure class="highlight asciidoc">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Sun Mar 11 15:34:14 2018       </span><br><span class="line"><span class="code">+-----------------------------------------------------------------------------+</span></span><br><span class="line">| NVIDIA-SMI 384.111                Driver Version: 384.111                   |</span><br><span class="line">|-------------------------------<span class="code">+----------------------+</span>----------------------+</span><br><span class="line">| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |</span><br><span class="line">| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |</span><br><span class="line">|===============================<span class="code">+======================+</span>======================|</span><br><span class="line">|   0  GeForce GTX 1080    Off  | 00000000:02:00.0  On |                  N/A |</span><br><span class="line">|  0%   36C    P8    15W / 320W |    224MiB /  8105MiB |      0%      Default |</span><br><span class="line"><span class="code">+-------------------------------+</span>----------------------<span class="code">+----------------------+</span></span><br><span class="line"><span class="code">                                                                               </span></span><br><span class="line"><span class="code">+-----------------------------------------------------------------------------+</span></span><br><span class="line">| Processes:                                                       GPU Memory |</span><br><span class="line">|  GPU       PID   Type   Process name                             Usage      |</span><br><span class="line">|=============================================================================|</span><br><span class="line">|    0      1322      G   /usr/lib/xorg/Xorg                           178MiB |</span><br><span class="line">|    0      1880      G   compiz                                        43MiB |</span><br><span class="line"><span class="code">+-----------------------------------------------------------------------------+</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="进程"><a href="#进程" class="headerlink" title="进程"></a>进程</h2>
                  <p>查看所有进程：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">ps -ef</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>示例结果：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">UID        PID  PPID  C STIME TTY          TIME CMD</span><br><span class="line">root         <span class="number">1</span>     <span class="number">0</span>  <span class="number">0</span> Feb09 ?        <span class="number">00</span>:<span class="number">00</span>:<span class="number">15</span> /sbin/init splash</span><br><span class="line">root         <span class="number">2</span>     <span class="number">0</span>  <span class="number">0</span> Feb09 ?        <span class="number">00</span>:<span class="number">00</span>:<span class="number">00</span> [kthreadd]</span><br><span class="line">root         <span class="number">3</span>     <span class="number">2</span>  <span class="number">0</span> Feb09 ?        <span class="number">00</span>:<span class="number">00</span>:<span class="number">05</span> [ksoftirqd/<span class="number">0</span>]</span><br><span class="line">root         <span class="number">5</span>     <span class="number">2</span>  <span class="number">0</span> Feb09 ?        <span class="number">00</span>:<span class="number">00</span>:<span class="number">00</span> [kworker/<span class="number">0</span>:<span class="number">0</span>H]</span><br><span class="line">root         <span class="number">6</span>     <span class="number">2</span>  <span class="number">0</span> Feb09 ?        <span class="number">00</span>:<span class="number">00</span>:<span class="number">01</span> [kworker/u96:<span class="number">0</span>]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>筛选进程：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">ps</span> -ef | <span class="keyword">grep</span> <span class="keyword">python</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>示例结果：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">chen      <span class="number">6466</span> <span class="number">44495</span>  <span class="number">0</span> Mar09 ?        <span class="number">00</span>:<span class="number">01</span>:<span class="number">56</span> /usr/bin/python2 -m ipykernel_launcher -f /run/user/<span class="number">1000</span>/jupyter/kernel<span class="number">-218926</span>a1-f3b9<span class="number">-4410</span>-bffb<span class="number">-1</span>de1341732be.json</span><br><span class="line">chen     <span class="number">11630</span> <span class="number">44495</span>  <span class="number">0</span> Mar09 ?        <span class="number">00</span>:<span class="number">00</span>:<span class="number">13</span> /home/chen/anaconda3/bin/python -m ipykernel_launcher -f /run/user/<span class="number">1000</span>/jupyter/kernel-c2942b7d-d73c<span class="number">-420</span>a-b6be<span class="number">-78f</span>0169e68c9.json</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>根据名称强制杀死进程：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">ps -ef | grep python | cut -c <span class="number">9</span><span class="number">-15</span> | xargs kill <span class="number">-9</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="用户"><a href="#用户" class="headerlink" title="用户"></a>用户</h2>
                  <p>添加用户：</p>
                  <figure class="highlight armasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">sudo</span> <span class="keyword">adduser </span>username</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>添加用户并设置目录：</p>
                  <figure class="highlight arduino">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo adduser username --<span class="built_in">home</span> /<span class="built_in">home</span>/username</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>将用户设置超级用户组（如添加到 sudo 用户组）：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">sudo usermod -aG sudo username</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>查看所有用户组：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">groups</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>查看某个用户组下所有用户：</p>
                  <figure class="highlight properties">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">sudo</span> <span class="string">apt-get install members</span></span><br><span class="line"><span class="attr">members</span> <span class="string">sudo</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-03-11 15:35:40" itemprop="dateCreated datePublished" datetime="2018-03-11T15:35:40+08:00">2018-03-11</time>
                </span>
                <span id="/5806.html" class="post-meta-item leancloud_visitors" data-flag-title="Linux常用命令总结" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>4.3k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>4 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5788.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5788.html" class="post-title-link" itemprop="url">正则表达式中零宽断言的用法</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>了解了正则表达式，想必一般情况下的匹配都不会出现什么问题，但是如果一些特殊情况，可能需要用到一些更高级的正则表达式匹配操作，本节我们来说明一下正则表达式的一个较常用又比较重要的知识点——零宽断言。</p>
                  <h2 id="实例引入"><a href="#实例引入" class="headerlink" title="实例引入"></a>实例引入</h2>
                  <p>首先我们来看一个例子，这里有一段问答对话：</p>
                  <blockquote>
                    <p>问：我用的是Windows XP+Service Pack 2，为什么无法安装输入卡号和密码的控件？ 答：在Windows XP+Service Pack 2、Windows 2003等操作系统中，用户可以自己选择是否安装控件。 问：为什么我看到的卡号输入框显示为*符号？ 答：您的浏览器禁止下载执行ActiveX控件 , 对于这种情况 , 您必须打开浏览器的ActiveX的相关权限。 操作方法：在浏览器菜单中选择“工具”|“Internet选项”，在弹出的对话框中选择”安全” |”Internet”|”自定义级别”，在弹出的对话框中选择”重置为 安全级-中” , 点”重置”按钮，确定。 问：看了以上几个问题，还是不能登录，怎么办？ 答：您的浏览器由于其他原因不能安装招商银行登录控件， 请下载并安装招商银行登录控件下载版。 问：无法出现个人网上银行大众版登录界面。 答：这种情况是由于您的机器无法和我行服务器建立安全连接，通常是因为代理服务器设置错误引起。如果您是拨号上网，请不要使用代理服务器；如果您过去安装过我行SSL安全代理，请调用“添加-删除程序”删除SSL安全代理；如果您是经过代理访问Internet，请联系您所在网的网络管理员设置代理服务器。IE5.0浏览器设置代理服务器的步骤： Internet选项—&gt;连接—&gt;局域网设置—&gt;使用代理服务器—&gt;高级。 问：我在输入账号和卡号时，总出错，该怎样输？ 答：存折账号为10位，按存折本上的账号输入， 密码为6位。如果一卡通是12位卡号的，只需输入地区码后面的8位卡号，不需要输入前面4位的地区码，密码为6位。如果一卡通是16位卡号的，请将16位卡号全部输入，密码为6位。 问：我的存折没有设密码，怎样在个人网上银行大众版中查询余额？ 答：存折必须设有密码方可在 个人网上银行大众版 中查询，因此请您到存折开户行给您的存折设置密码。 注：网上个人银行是招商银行为个人客户提供的网上银行。 本页面内容仅供参考，部分业务以当地网点的公告与具体规定为准。</p>
                  </blockquote>
                  <p>我们需要将这段对话中的问题和答案对提取出来，即提取出如下内容：</p>
                  <blockquote>
                    <p>Q：我用的是Windows XP+Service Pack 2，为什么无法安装输入卡号和密码的控件？ A：在Windows XP+Service Pack 2、Windows 2003等操作系统中，用户可以自己选择是否安装控件。 Q：为什么我看到的卡号输入框显示为*符号？ A：您的浏览器禁止下载执行ActiveX控件 , 对于这种情况 , 您必须打开浏览器的ActiveX的相关权限。 操作方法：在浏览器菜单中选择“工具”|“Internet选项”，在弹出的对话框中选择”安全” |”Internet”|”自定义级别”，在弹出的对话框中选择”重置为 安全级-中” , 点”重置”按钮，确定。 …</p>
                  </blockquote>
                  <p>如果要用 Python 实现的话，那么我们很可能自然而然想到 split() 或 findall() 方法，如果用 split() 方法，我们可能会这么写：</p>
                  <figure class="highlight perl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import re</span><br><span class="line">results = re.<span class="keyword">split</span>(<span class="string">'问：| 答：'</span>, text)</span><br><span class="line"><span class="keyword">for</span> <span class="keyword">index</span>, result in enumerate(results[<span class="number">1</span>:]):</span><br><span class="line">    <span class="keyword">print</span>((<span class="string">'Q'</span> <span class="keyword">if</span> <span class="keyword">index</span>%2 == <span class="number">0</span> <span class="keyword">else</span> <span class="string">'A'</span>) + <span class="string">': '</span> + result)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里 split() 方法的第一个参数传入了 <code>问：| 答：</code> 这个正则表达式，意思是将这段话用 <code>问：</code> 或者 <code>答：</code> 分开，这个功能是正则表达式对字符串进行分割的方法，相比直接字符串的 split() 方法功能更为强大。这里其实得到的结果是一个列表，长度是一个奇数，如果我们把 results 打印出来，结果是这样的：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">['', <span class="symbol">'我用的是Windows</span> XP+Service Pack <span class="number">2</span>，为什么无法安装输入卡号和密码的控件？', <span class="symbol">'在Windows</span> XP+Service Pack <span class="number">2</span>、Windows <span class="number">2003</span>等操作系统中，用户可以自己选择是否安装控件。 ', <span class="symbol">'为什么我看到的卡号输入框显示为*符号？</span>', <span class="symbol">'您的浏览器禁止下载执行ActiveX控件</span> , 对于这种情况 , 您必须打开浏览器的ActiveX的相关权限。 操作方法：在浏览器菜单中选择“工具”|“Internet选项”，在弹出的对话框中选择<span class="string">"安全"</span> |<span class="string">"Internet"</span>|<span class="string">"自定义级别"</span>，在弹出的对话框中选择<span class="string">"重置为 安全级-中"</span> , 点<span class="string">"重置"</span>按钮，确定。 ', <span class="symbol">'看了以上几个问题，还是不能登录，怎么办？</span>', <span class="symbol">'您的浏览器由于其他原因不能安装招商银行登录控件，</span> 请下载并安装招商银行登录控件下载版。 ', <span class="symbol">'无法出现个人网上银行大众版登录界面。</span>', <span class="symbol">'这种情况是由于您的机器无法和我行服务器建立安全连接，通常是因为代理服务器设置错误引起。如果您是拨号上网，请不要使用代理服务器；如果您过去安装过我行SSL安全代理，请调用“添加-删除程序”删除SSL安全代理；如果您是经过代理访问Internet，请联系您所在网的网络管理员设置代理服务器。IE5.0浏览器设置代理服务器的步骤：</span> Internet选项--&gt;连接--&gt;局域网设置--&gt;使用代理服务器--&gt;高级。 ', <span class="symbol">'我在输入账号和卡号时，总出错，该怎样输？</span>', <span class="symbol">'存折账号为10位，按存折本上的账号输入，</span> 密码为6位。如果一卡通是12位卡号的，只需输入地区码后面的8位卡号，不需要输入前面4位的地区码，密码为6位。如果一卡通是16位卡号的，请将16位卡号全部输入，密码为6位。 ', <span class="symbol">'我的存折没有设密码，怎样在个人网上银行大众版中查询余额？</span>', <span class="symbol">'存折必须设有密码方可在</span> 个人网上银行大众版 中查询，因此请您到存折开户行给您的存折设置密码。 注：网上个人银行是招商银行为个人客户提供的网上银行。 本页面内容仅供参考，部分业务以当地网点的公告与具体规定为准。 ']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这是因为我们分割使用的字符本身就处于整个文本的字符，所以一上来就找到了分割的标志 <code>问：</code>，所以它左侧的结果就是空字符串了，所以最终得到的结果第一个内容就是空字符串，后续的内容便是正常的一问一答的短句。所以这里我们还需要对结果进行切片操作，去除第一个元素，然后将其遍历打印输出，最终结果如下：</p>
                  <figure class="highlight avrasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">Q:</span> 我用的是Windows XP+Service Pack <span class="number">2</span>，为什么无法安装输入卡号和密码的控件？</span><br><span class="line"><span class="symbol">A:</span> 在Windows XP+Service Pack <span class="number">2</span>、Windows <span class="number">2003</span>等操作系统中，用户可以自己选择是否安装控件。 </span><br><span class="line"><span class="symbol">Q:</span> 为什么我看到的卡号输入框显示为*符号？</span><br><span class="line"><span class="symbol">A:</span> 您的浏览器禁止下载执行ActiveX控件 , 对于这种情况 , 您必须打开浏览器的ActiveX的相关权限。 操作方法：在浏览器菜单中选择“工具”|“Internet选项”，在弹出的对话框中选择<span class="string">"安全"</span> |<span class="string">"Internet"</span>|<span class="string">"自定义级别"</span>，在弹出的对话框中选择<span class="string">"重置为 安全级-中"</span> , 点<span class="string">"重置"</span>按钮，确定。 </span><br><span class="line"><span class="symbol">Q:</span> 看了以上几个问题，还是不能登录，怎么办？</span><br><span class="line"><span class="symbol">A:</span> 您的浏览器由于其他原因不能安装招商银行登录控件， 请下载并安装招商银行登录控件下载版。 </span><br><span class="line"><span class="symbol">Q:</span> 无法出现个人网上银行大众版登录界面。</span><br><span class="line"><span class="symbol">A:</span> 这种情况是由于您的机器无法和我行服务器建立安全连接，通常是因为代理服务器设置错误引起。如果您是拨号上网，请不要使用代理服务器；如果您过去安装过我行SSL安全代理，请调用“添加-删除程序”删除SSL安全代理；如果您是经过代理访问Internet，请联系您所在网的网络管理员设置代理服务器。IE5<span class="number">.0</span>浏览器设置代理服务器的步骤： Internet选项--&gt;连接--&gt;局域网设置--&gt;使用代理服务器--&gt;高级。 </span><br><span class="line"><span class="symbol">Q:</span> 我在输入账号和卡号时，总出错，该怎样输？</span><br><span class="line"><span class="symbol">A:</span> 存折账号为<span class="number">10</span>位，按存折本上的账号输入， 密码为<span class="number">6</span>位。如果一卡通是<span class="number">12</span>位卡号的，只需输入地区码后面的<span class="number">8</span>位卡号，不需要输入前面<span class="number">4</span>位的地区码，密码为<span class="number">6</span>位。如果一卡通是<span class="number">16</span>位卡号的，请将<span class="number">16</span>位卡号全部输入，密码为<span class="number">6</span>位。 </span><br><span class="line"><span class="symbol">Q:</span> 我的存折没有设密码，怎样在个人网上银行大众版中查询余额？</span><br><span class="line"><span class="symbol">A:</span> 存折必须设有密码方可在 个人网上银行大众版 中查询，因此请您到存折开户行给您的存折设置密码。 注：网上个人银行是招商银行为个人客户提供的网上银行。 本页面内容仅供参考，部分业务以当地网点的公告与具体规定为准。</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样确实没问题，我们可以顺利地提取出来，但是总感觉这个解法并不那么优雅，因为我们这里是将问题和答案的内容都单独切出来了，并没有将问答对一块提取，而且 split() 方法返回的结果的第一个元素还不是我们想要的结果，所以还需要进行一些切片操作来去除，所以整个写法感觉实现起来并不完美。 所以我们又想到了 findall() 方法，这时我们会这么写：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import re</span><br><span class="line">results = re.findall(<span class="string">'问：(.*?) 答：(.*?)'</span>, <span class="keyword">text</span>, re.S)</span><br><span class="line"><span class="keyword">for</span> <span class="built_in">result</span> <span class="keyword">in</span> results:</span><br><span class="line">    print(<span class="string">'Q: '</span> + <span class="built_in">result</span>[<span class="number">0</span>], <span class="string">'A: '</span> + <span class="built_in">result</span>[<span class="number">1</span>], sep=<span class="string">'\n'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>表面上看似乎是把问题答案对用正则表示出来了，而且使用了非贪婪匹配，但是很明显，在末尾我们并没有指定匹配的终点，所以整个的结果就会导致回答是完全匹配不到的，运行结果如下：</p>
                  <figure class="highlight yaml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">Q:</span> <span class="string">我用的是Windows</span> <span class="string">XP+Service</span> <span class="string">Pack</span> <span class="number">2</span><span class="string">，为什么无法安装输入卡号和密码的控件？</span></span><br><span class="line"><span class="attr">A:</span> </span><br><span class="line"><span class="attr">Q:</span> <span class="string">为什么我看到的卡号输入框显示为*符号？</span></span><br><span class="line"><span class="attr">A:</span> </span><br><span class="line"><span class="attr">Q:</span> <span class="string">看了以上几个问题，还是不能登录，怎么办？</span></span><br><span class="line"><span class="attr">A:</span> </span><br><span class="line"><span class="attr">Q:</span> <span class="string">无法出现个人网上银行大众版登录界面。</span></span><br><span class="line"><span class="attr">A:</span> </span><br><span class="line"><span class="attr">Q:</span> <span class="string">我在输入账号和卡号时，总出错，该怎样输？</span></span><br><span class="line"><span class="attr">A:</span> </span><br><span class="line"><span class="attr">Q:</span> <span class="string">我的存折没有设密码，怎样在个人网上银行大众版中查询余额？</span></span><br><span class="line"><span class="attr">A:</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>好，那么我们加上匹配的终点吧，以下一个的 <code>问：</code> 作为我们正则表达式匹配的终点总可以了吧？所以我们可能会改写成这样子：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import re</span><br><span class="line">results = re.findall(<span class="string">'问：(.*?) 答：(.*?)问：'</span>, <span class="keyword">text</span>, re.S)</span><br><span class="line"><span class="keyword">for</span> <span class="built_in">result</span> <span class="keyword">in</span> results:</span><br><span class="line">    print(<span class="string">'Q: '</span> + <span class="built_in">result</span>[<span class="number">0</span>], <span class="string">'A: '</span> + <span class="built_in">result</span>[<span class="number">1</span>], sep=<span class="string">'\n'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样写似乎看起来是可以了，但结果却是这样的：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Q: 我用的是Windows XP+Service Pack <span class="number">2</span>，为什么无法安装输入卡号和密码的控件？</span><br><span class="line">A: 在Windows XP+Service Pack <span class="number">2</span>、Windows <span class="number">2003</span>等操作系统中，用户可以自己选择是否安装控件。 </span><br><span class="line">Q: 看了以上几个问题，还是不能登录，怎么办？</span><br><span class="line">A: 您的浏览器由于其他原因不能安装招商银行登录控件， 请下载并安装招商银行登录控件下载版。 </span><br><span class="line">Q: 我在输入账号和卡号时，总出错，该怎样输？</span><br><span class="line">A: 存折账号为<span class="number">10</span>位，按存折本上的账号输入， 密码为<span class="number">6</span>位。如果一卡通是<span class="number">12</span>位卡号的，只需输入地区码后面的<span class="number">8</span>位卡号，不需要输入前面<span class="number">4</span>位的地区码，密码为<span class="number">6</span>位。如果一卡通是<span class="number">16</span>位卡号的，请将<span class="number">16</span>位卡号全部输入，密码为<span class="number">6</span>位。</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果只剩三个问题答案对了，有三个问答对被“吃”掉了，其实这是因为我们的正则表达式最后加了 <code>问:</code>的缘故，findall() 方法它会查找所有符合正则表达式的结果，但其中匹配的时候它内部也是有一个查找索引在扫描的。在查找第一个符合要求的结果时，由于我们是根据正则表达式结尾的 <code>问：</code>来作为结束标志，所以在找到第一个符合要求的结果时，我们的查找索引就已经移动到了第二个问答对开头的 <code>问：</code> 上面，即查找索引就已经进入到了第二个问答对的位置了，而在下一次查找符合要求的结果时，索引会继续往后移动进行扫描，所以它是从第二个问答对的 <code>问：</code> 后面继续扫描的，所以对于第二个问答对，实际上已经被割裂了，所以它只能查找到第三个问答对的时候才可以发现符合正则表达式的内容。因此，我们可以观察到，返回的结果只是第一、三、五三个问答对。 所以，如果我们想要用该方法找到完整的留个问答对，就需要用到零宽断言了。 解法如下：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import re</span><br><span class="line">results = re.findall(<span class="string">'问：(.*?) 答：(.*?)(?=问：|\Z)'</span>, <span class="keyword">text</span>, re.S)</span><br><span class="line"><span class="keyword">for</span> <span class="built_in">result</span> <span class="keyword">in</span> results:</span><br><span class="line">    print(<span class="string">'Q: '</span> + <span class="built_in">result</span>[<span class="number">0</span>], <span class="string">'A: '</span> + <span class="built_in">result</span>[<span class="number">1</span>], sep=<span class="string">'\n'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight avrasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">Q:</span> 我用的是Windows XP+Service Pack <span class="number">2</span>，为什么无法安装输入卡号和密码的控件？</span><br><span class="line"><span class="symbol">A:</span> 在Windows XP+Service Pack <span class="number">2</span>、Windows <span class="number">2003</span>等操作系统中，用户可以自己选择是否安装控件。 </span><br><span class="line"><span class="symbol">Q:</span> 为什么我看到的卡号输入框显示为*符号？</span><br><span class="line"><span class="symbol">A:</span> 您的浏览器禁止下载执行ActiveX控件 , 对于这种情况 , 您必须打开浏览器的ActiveX的相关权限。 操作方法：在浏览器菜单中选择“工具”|“Internet选项”，在弹出的对话框中选择<span class="string">"安全"</span> |<span class="string">"Internet"</span>|<span class="string">"自定义级别"</span>，在弹出的对话框中选择<span class="string">"重置为 安全级-中"</span> , 点<span class="string">"重置"</span>按钮，确定。 </span><br><span class="line"><span class="symbol">Q:</span> 看了以上几个问题，还是不能登录，怎么办？</span><br><span class="line"><span class="symbol">A:</span> 您的浏览器由于其他原因不能安装招商银行登录控件， 请下载并安装招商银行登录控件下载版。 </span><br><span class="line"><span class="symbol">Q:</span> 无法出现个人网上银行大众版登录界面。</span><br><span class="line"><span class="symbol">A:</span> 这种情况是由于您的机器无法和我行服务器建立安全连接，通常是因为代理服务器设置错误引起。如果您是拨号上网，请不要使用代理服务器；如果您过去安装过我行SSL安全代理，请调用“添加-删除程序”删除SSL安全代理；如果您是经过代理访问Internet，请联系您所在网的网络管理员设置代理服务器。IE5<span class="number">.0</span>浏览器设置代理服务器的步骤： Internet选项--&gt;连接--&gt;局域网设置--&gt;使用代理服务器--&gt;高级。 </span><br><span class="line"><span class="symbol">Q:</span> 我在输入账号和卡号时，总出错，该怎样输？</span><br><span class="line"><span class="symbol">A:</span> 存折账号为<span class="number">10</span>位，按存折本上的账号输入， 密码为<span class="number">6</span>位。如果一卡通是<span class="number">12</span>位卡号的，只需输入地区码后面的<span class="number">8</span>位卡号，不需要输入前面<span class="number">4</span>位的地区码，密码为<span class="number">6</span>位。如果一卡通是<span class="number">16</span>位卡号的，请将<span class="number">16</span>位卡号全部输入，密码为<span class="number">6</span>位。 </span><br><span class="line"><span class="symbol">Q:</span> 我的存折没有设密码，怎样在个人网上银行大众版中查询余额？</span><br><span class="line"><span class="symbol">A:</span> 存折必须设有密码方可在 个人网上银行大众版 中查询，因此请您到存折开户行给您的存折设置密码。 注：网上个人银行是招商银行为个人客户提供的网上银行。 本页面内容仅供参考，部分业务以当地网点的公告与具体规定为准。</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们实际上是使用了 <code>(?=)</code>这样的形式来构建了整个表达式，等号后面的内容是 <code>问：</code>或者结束符 <code>\\Z</code>，这样其实就保证了在匹配的时候，查找索引不会继续向后移，但这也同时标志了结束标志，因此它就可以查找到完整的内容了。</p>
                  <h2 id="零宽断言"><a href="#零宽断言" class="headerlink" title="零宽断言"></a>零宽断言</h2>
                  <p>零宽断言，顾名思义，是一种零宽度的匹配，它匹配的内容不会保存到匹配结果中，表达式的匹配内容只是代表了一个位置而已，如标明某个字符的右边界是怎样的构造。 在前面我们使用了 <code>?=</code>来进行了实例讲解，这是其中一个用法，另外还有 <code>?&lt;=</code>、<code>?!</code>、<code>?&lt;!</code>，下面我们来依次进行讲解说明。</p>
                  <ul>
                    <li><code>?=</code>代表零宽度正预测先行断言，它断言自身出现的位置的后面可以匹配后面跟的表达式。</li>
                    <li><code>?&lt;=</code>代表零宽度正回顾后发断言，它断言自身出现的位置的前面可以匹配后面跟的表达式。</li>
                    <li><code>?!</code>代表零宽度负预测先行断言，它断言自身出现的位置的后面不可以匹配后面跟的表达式。</li>
                    <li><code>?&lt;!</code>代表零宽度负回顾后发断言，它断言自身出现的位置的后面不可以匹配后面跟的表达式。</li>
                  </ul>
                  <h3 id=""><a href="#" class="headerlink" title="?="></a>?=</h3>
                  <p>首先我们来看下 <code>?=</code>的用法，它断言自身出现的位置的后面可以匹配后面跟的表达式。 比如我们这里有这样的一个字符串：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">str</span> = <span class="string">'我的个人邮箱是cqc@cuiqingcai.com，个人博客是cuiqingcai.com，个人公众号是进击的Coder'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里我们想把我的个人邮箱这句话和个人邮箱单独摘出来，假如我们不使用零宽断言的话，我们需要给个人邮箱后面这一句加一个结束标识符或者单独匹配邮箱作为标识符，我们可能会这么写：</p>
                  <figure class="highlight axapta">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import re</span><br><span class="line"><span class="keyword">str</span> = <span class="string">'我的个人邮箱是cqc@cuiqingcai.com，个人博客是cuiqingcai.com，个人公众号是进击的Coder'</span></span><br><span class="line">result = re.search(<span class="string">'我的个人邮箱是(.*?)，个人博客'</span>, <span class="keyword">str</span>)</span><br><span class="line">print(<span class="string">'整句结果：'</span> + result.<span class="keyword">group</span>(), <span class="string">'第一个匹配结果：'</span> + result.<span class="keyword">group</span>(<span class="number">1</span>), sep=<span class="string">'\n'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在正则表达式的最后我们加了<code>，个人博客</code>作为匹配的结束符，然后邮箱部分用非贪婪匹配的模式进行匹配，我们看下运行结果：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">整句结果：我的个人邮箱是<span class="symbol">cqc@</span>cuiqingcai.com，个人博客</span><br><span class="line">第一个匹配结果：<span class="symbol">cqc@</span>cuiqingcai.com</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们可以看到第一个匹配结果成功得到了邮箱信息，但是我们看整句结果缺并不理想，它多匹配了我们加入的结尾标识，并没有得到正常的一句话。 这时候如果我们改用 <code>?=</code>来匹配，结果就不会带有此标识符了，改写如下：</p>
                  <figure class="highlight axapta">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import re</span><br><span class="line"><span class="keyword">str</span> = <span class="string">'我的个人邮箱是cqc@cuiqingcai.com，个人博客是cuiqingcai.com，个人公众号是进击的Coder'</span></span><br><span class="line">result = re.search(<span class="string">'我的个人邮箱是(.*?)(?=，个人博客)'</span>, <span class="keyword">str</span>)</span><br><span class="line">print(<span class="string">'整句结果：'</span> + result.<span class="keyword">group</span>(), <span class="string">'第一个匹配结果：'</span> + result.<span class="keyword">group</span>(<span class="number">1</span>), sep=<span class="string">'\n'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里我们将结尾标识符改成了 <code>(?=，个人博客)</code> ，这样就将此部分内容作为零宽度匹配，它代表后面需要跟 <code>，个人博客</code>，但是它不会出现在匹配结果中。 运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">整句结果：我的个人邮箱是<span class="symbol">cqc@</span>cuiqingcai.com</span><br><span class="line">第一个匹配结果：<span class="symbol">cqc@</span>cuiqingcai.com</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到整句结果中已经没有无用的后缀字符了。</p>
                  <h3 id="lt"><a href="#lt" class="headerlink" title="?&lt;="></a>?&lt;=</h3>
                  <p>接下来我们再看下 <code>?&lt;=</code>的用法，它代表零宽度正回顾后发断言，其实就是匹配前面的标识，比如这里我们还是以上面的例子为例，匹配出个人博客这句话，代码如下：</p>
                  <figure class="highlight axapta">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import re</span><br><span class="line"><span class="keyword">str</span> = <span class="string">'我的个人邮箱是cqc@cuiqingcai.com，个人博客是cuiqingcai.com，个人公众号是进击的Coder'</span></span><br><span class="line">result = re.search(<span class="string">'(?&lt;=，)个人博客是(.*?)(?=，)'</span>, <span class="keyword">str</span>)</span><br><span class="line">print(<span class="string">'整句结果：'</span> + result.<span class="keyword">group</span>(), <span class="string">'第一个匹配结果：'</span> + result.<span class="keyword">group</span>(<span class="number">1</span>), sep=<span class="string">'\n'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们在<code>个人博客</code> 前面加了一个零宽断言的逗号符号作为开头，使用的就是 <code>?&lt;=</code>，句子结尾是用的 <code>?=</code>，这样前后的标识都不会匹配到了，运行结果如下：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">整句结果：个人博客是<span class="selector-tag">cuiqingcai</span><span class="selector-class">.com</span></span><br><span class="line">第一个匹配结果：<span class="selector-tag">cuiqingcai</span><span class="selector-class">.com</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到得到的整句结果也是完整的一句话。</p>
                  <h3 id="-1"><a href="#-1" class="headerlink" title="?!"></a>?!</h3>
                  <p><code>?!</code>代表代表零宽度负预测先行断言，它断言自身出现的位置的后面不可以匹配后面跟的表达式。也是用来匹配后面的文本，但这里是取反，它指定了后面出现的内容不匹配该标识，我们在前面的例子基础上修改如下：</p>
                  <figure class="highlight axapta">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import re</span><br><span class="line"><span class="keyword">str</span> = <span class="string">'我的个人邮箱是cqc@cuiqingcai.com，个人博客是cuiqingcai.com，个人公众号是进击的Coder'</span></span><br><span class="line">result = re.search(<span class="string">'我的个人邮箱是(.*?)(?!，个人公众号)(?=，个人博客)'</span>, <span class="keyword">str</span>)</span><br><span class="line">print(<span class="string">'整句结果：'</span> + result.<span class="keyword">group</span>(), <span class="string">'第一个匹配结果：'</span> + result.<span class="keyword">group</span>(<span class="number">1</span>), sep=<span class="string">'\n'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>本来是 <code>(?=，个人博客)</code>的标识符，不过这里我们使用 <code>?!</code>来指定了另一个标识符<code>，个人公众号</code>，这就代表这句话后面跟的需要是<code>(?=，个人博客)</code>而不是<code>，个人公众号</code>，运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">整句结果：我的个人邮箱是<span class="symbol">cqc@</span>cuiqingcai.com</span><br><span class="line">第一个匹配结果：<span class="symbol">cqc@</span>cuiqingcai.com</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="lt-1"><a href="#lt-1" class="headerlink" title="?&lt;!"></a>?&lt;!</h3>
                  <p><code>?&lt;!</code>代表零宽度负回顾后发断言，它断言自身出现的位置的后面不可以匹配后面跟的表达式。我们在前面的例子基础上加以修改：</p>
                  <figure class="highlight axapta">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import re</span><br><span class="line"><span class="keyword">str</span> = <span class="string">'我的个人邮箱是cqc@cuiqingcai.com，个人博客是cuiqingcai.com，个人公众号是进击的Coder'</span></span><br><span class="line">result = re.search(<span class="string">'(?&lt;=，)(?&lt;!。)个人博客是(.*?)(?=，)'</span>, <span class="keyword">str</span>)</span><br><span class="line">print(<span class="string">'整句结果：'</span> + result.<span class="keyword">group</span>(), <span class="string">'第一个匹配结果：'</span> + result.<span class="keyword">group</span>(<span class="number">1</span>), sep=<span class="string">'\n'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们写了 <code>?&lt;!</code>标识符，后面跟了一个句号，这代表前面不应该出现句号。 运行结果如下：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">整句结果：个人博客是<span class="selector-tag">cuiqingcai</span><span class="selector-class">.com</span></span><br><span class="line">第一个匹配结果：<span class="selector-tag">cuiqingcai</span><span class="selector-class">.com</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="常用用法"><a href="#常用用法" class="headerlink" title="常用用法"></a>常用用法</h2>
                  <p>其实上面的示例中我们使用了 search() 方法进行了内容匹配，其实这并不常用，因为一般我们更关注的是匹配分组结果的内容，其实更多的用法是用在了 findall() 方法上，它用来匹配多个结果，也就类似于我们一开始的实例一样，这里我们还是以刚才的字符串为例，来输出一下个人邮箱、个人博客、个人公众号三个内容，代码如下：</p>
                  <figure class="highlight processing">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> re</span><br><span class="line"><span class="built_in">str</span> = <span class="string">'我的个人邮箱是cqc@cuiqingcai.com，个人博客是cuiqingcai.com，个人公众号是进击的Coder'</span></span><br><span class="line">results = re.findall(<span class="string">'个人(.*?)是(.*?)(?=，|\Z)'</span>, <span class="built_in">str</span>)</span><br><span class="line"><span class="keyword">for</span> result in results:</span><br><span class="line">    <span class="built_in">print</span>(result[<span class="number">0</span>] + <span class="string">': '</span> + result[<span class="number">1</span>])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们匹配了<code>个人</code>二字，然后后面跟了非贪婪匹配，然后加了一个<code>是</code>字，最关键的是结尾标识符，这里必须要使用零宽断言才可以匹配出三个结果，这里匹配的内容是 <code>，|\\Z</code>，意思是匹配逗号或结束符。 运行结果如下：</p>
                  <figure class="highlight makefile">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="section">邮箱: cqc@cuiqingcai.com</span></span><br><span class="line"><span class="section">博客: cuiqingcai.com</span></span><br><span class="line"><span class="section">公众号: 进击的Coder</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们就成功输出了邮箱、博客及公众号的内容了，匹配非常顺利方便。</p>
                  <h2 id="结语"><a href="#结语" class="headerlink" title="结语"></a>结语</h2>
                  <p>通过本节，我们应该大体可以了解了正则表达式中零宽断言的基本用法和适用场景，相信理解了零宽断言之后，我们再做正则匹配时会更加得心应手。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-03-03 05:42:18" itemprop="dateCreated datePublished" datetime="2018-03-03T05:42:18+08:00">2018-03-03</time>
                </span>
                <span id="/5788.html" class="post-meta-item leancloud_visitors" data-flag-title="正则表达式中零宽断言的用法" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>9.1k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>8 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5771.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5771.html" class="post-title-link" itemprop="url">机器学习主要术语</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>什么是（监督式）机器学习？简单来说，它的定义如下：</p>
                  <ul>
                    <li>机器学习系统通过学习如何组合输入信息来对从未见过的数据做出有用的预测。</li>
                  </ul>
                  <p>下面我们来了解一下机器学习的基本术语。</p>
                  <h2 id="标签"><a href="#标签" class="headerlink" title="标签"></a>标签</h2>
                  <p>在简单线性回归中，标签是我们要预测的事物，即 y 变量。标签可以是小麦未来的价格、图片中显示的动物品种、音频剪辑的含义或任何事物。</p>
                  <h2 id="特征"><a href="#特征" class="headerlink" title="特征"></a>特征</h2>
                  <p>在简单线性回归中，特征是输入变量，即 x 变量。简单的机器学习项目可能会使用单个特征，而比较复杂的机器学习项目可能会使用数百万个特征，按如下方式指定：</p>
                  <p>{x1,x2,…xN}</p>
                  <p>在垃圾邮件检测器示例中，特征可能包括：</p>
                  <ul>
                    <li>电子邮件文本中的字词</li>
                    <li>发件人的地址</li>
                    <li>发送电子邮件的时段</li>
                    <li>电子邮件中包含“一种奇怪的把戏”这样的短语。</li>
                  </ul>
                  <h2 id="样本"><a href="#样本" class="headerlink" title="样本"></a>样本</h2>
                  <p>样本是指数据的特定实例：x。（我们采用粗体 x 表示它是一个矢量。）我们将样本分为以下两类：</p>
                  <ul>
                    <li>有标签样本</li>
                    <li>无标签样本</li>
                  </ul>
                  <p>有标签样本同时包含特征和标签。即：</p>
                  <figure class="highlight dockerfile">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">  labeled examples: &#123;features, <span class="keyword">label</span><span class="bash">&#125;: (x, y)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们使用有标签样本来训练模型。在我们的垃圾邮件检测器示例中，有标签样本是用户明确标记为“垃圾邮件”或“非垃圾邮件”的各个电子邮件。 例如，下表显示了从包含加利福尼亚州房价信息的<a href="https://developers.google.cn/machine-learning/crash-course/california-housing-data-description" target="_blank" rel="noopener">数据集</a>中抽取的 5 个有标签样本：</p>
                  <p>housingMedianAge （特征）</p>
                  <p>totalRooms （特征）</p>
                  <p>totalBedrooms （特征）</p>
                  <p>medianHouseValue （标签）</p>
                  <p>15</p>
                  <p>5612</p>
                  <p>1283</p>
                  <p>66900</p>
                  <p>19</p>
                  <p>7650</p>
                  <p>1901</p>
                  <p>80100</p>
                  <p>17</p>
                  <p>720</p>
                  <p>174</p>
                  <p>85700</p>
                  <p>14</p>
                  <p>1501</p>
                  <p>337</p>
                  <p>73400</p>
                  <p>20</p>
                  <p>1454</p>
                  <p>326</p>
                  <p>65500</p>
                  <p>无标签样本包含特征，但不包含标签。即：</p>
                  <figure class="highlight groovy">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">  unlabeled <span class="string">examples:</span> &#123;features, ?&#125;: (x, ?)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在使用有标签样本训练了我们的模型之后，我们会使用该模型来预测无标签样本的标签。在垃圾邮件检测器示例中，无标签样本是用户尚未添加标签的新电子邮件。</p>
                  <h2 id="模型"><a href="#模型" class="headerlink" title="模型"></a>模型</h2>
                  <p>模型定义了特征与标签之间的关系。例如，垃圾邮件检测模型可能会将某些特征与“垃圾邮件”紧密联系起来。我们来重点介绍一下模型生命周期的两个阶段：</p>
                  <ul>
                    <li>训练表示创建或学习模型。也就是说，您向模型展示有标签样本，让模型逐渐学习特征与标签之间的关系。</li>
                    <li>推断表示将训练后的模型应用于无标签样本。也就是说，您使用训练后的模型来做出有用的预测 (y’)。例如，在推断期间，您可以针对新的无标签样本预测 medianHouseValue。</li>
                  </ul>
                  <h2 id="回归与分类"><a href="#回归与分类" class="headerlink" title="回归与分类"></a>回归与分类</h2>
                  <p>回归模型可预测连续值。例如，回归模型做出的预测可回答如下问题：</p>
                  <ul>
                    <li>加利福尼亚州一栋房产的价值是多少？</li>
                    <li>用户点击此广告的概率是多少？</li>
                  </ul>
                  <p>分类模型可预测离散值。例如，分类模型做出的预测可回答如下问题：</p>
                  <ul>
                    <li>某个指定电子邮件是垃圾邮件还是非垃圾邮件？</li>
                    <li>这是一张狗、猫还是仓鼠图片？</li>
                  </ul>
                  <h2 id="原文地址"><a href="#原文地址" class="headerlink" title="原文地址"></a>原文地址</h2>
                  <p>本节原文地址：<a href="https://developers.google.cn/machine-learning/crash-course/framing/ml-terminology" target="_blank" rel="noopener">问题构建 (Framing)：机器学习主要术语</a></p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-03-01 17:34:10" itemprop="dateCreated datePublished" datetime="2018-03-01T17:34:10+08:00">2018-03-01</time>
                </span>
                <span id="/5771.html" class="post-meta-item leancloud_visitors" data-flag-title="机器学习主要术语" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>1.2k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5737.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Paper <i class="label-arrow"></i>
                  </a>
                  <a href="/5737.html" class="post-title-link" itemprop="url">[论文笔记] Learning Phrase Representations using RNN Encoder–Decoder for Statistical Machine Translation</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>我们知道，Seq2Seq 现在已经成为了机器翻译、对话聊天、文本摘要等工作的重要模型，真正提出 Seq2Seq 的文章是《Sequence to Sequence Learning with Neural Networks》，但本篇《Learning Phrase Representations using RNN Encoder–Decoder for Statistical Machine Translation》比前者更早使用了 Seq2Seq 模型来解决机器翻译的问题，本文是该篇论文的概述。</p>
                  <h2 id="发布信息"><a href="#发布信息" class="headerlink" title="发布信息"></a>发布信息</h2>
                  <p><a href="https://arxiv.org/abs/1406.1078" target="_blank" rel="noopener">Learning Phrase Representations using RNN Encoder–Decoder for Statistical Machine Translation</a> 2014 年 6 月发布于 Arxiv 上，一作是 Kyunghyun Cho，当时来自蒙特利尔大学，现在在纽约大学任教。</p>
                  <h2 id="摘要"><a href="#摘要" class="headerlink" title="摘要"></a>摘要</h2>
                  <p>这篇论文中提出了一种新的模型，叫做 RNN Encoder-Decoder， 并将它用来进行机器翻译和比较不同语言的短语/词组之间的语义近似程度。这个模型由两个 RNN 组成，其中 Encoder 用来将输入的序列表示成一个固定长度的向量，Decoder 则使用这个向量重建出目标序列，另外该论文提出了 GRU 的基本结构，为后来的研究奠定了基础。 因此本文的主要贡献是：</p>
                  <ul>
                    <li>提出了一种类似于 LSTM 的 GRU 结构，并且具有比 LSTM 更少的参数，更不容易过拟合。</li>
                    <li>较早地将 Seq2Seq 应用在了机器翻译领域，并且取得了不错的效果。</li>
                  </ul>
                  <h2 id="模型"><a href="#模型" class="headerlink" title="模型"></a>模型</h2>
                  <p>本文提出的模型结构如下图所示： <img src="https://ws1.sinaimg.cn/large/006tNc79gy1foubbvos97j30iw0i8q4m.jpg" alt=""></p>
                  <p>这里首先对输入上文 x 走一遍 RNN，然后得到一个固定长度的向量 c，作为 Encoder，然后接下来再根据 c 和后续隐状态和输入状态来得到后续状态，Encoder 的行为比较简单，重点在 Decoder 上。</p>
                  <p>Decoder 中 t 时刻的内部状态的 ht 为:</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/WX20180227-003503@2x.png" alt=""></p>
                  <p>该时刻的输出概率则为: <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/WX20180227-003613@2x.png" alt=""> 模型训练时则去最大化给定输入序列 x 时输出序列为 y 的条件概率: <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/WX20180227-003657@2x.png" alt=""> 以上便是核心的公式，上面的这个就是该模型的优化目标。 在机器翻译上，作者用 Moses (一个 SMT 系统) 建立了一个 phrase based 的翻译模型作为 baseline system ，然后对比了以下四个模型的 BLEU 值</p>
                  <ol>
                    <li>Baseline configuration</li>
                    <li>Baseline + RNN</li>
                    <li>Baseline + CSLM + RNN</li>
                    <li>Baseline + CSLM + RNN + Word penalty</li>
                  </ol>
                  <p>四种不同的模型的 BLEU 值如下表所示：</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/WX20180227-005725@2x.png" alt=""></p>
                  <p>phrase pair 打分的结果如下： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/WX20180227-005736@2x.png" alt=""> 其中第一栏是输入的英语 phrase ，第二栏是用传统的模型得到的最近似的三个法语 phrase，第三栏是用 Encoder-Decoder 模型得到的最近似的三个 phrase。 另外作者还说明该模型可以学习到一个比较好的效果的 Word Embedding 结果，附图如下： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/WX20180227-003954@2x.png" alt=""> 左上角的图代表全局的 Embedding 结果，另外三个图是局部结果，可以看到类似的名词都被聚到了一起。</p>
                  <h2 id="GRU"><a href="#GRU" class="headerlink" title="GRU"></a>GRU</h2>
                  <p>另外本文的一个主要贡献是提出了 GRU 的门结构，相比 LSTM 更加简洁，而且效果不输 LSTM，关于它的详细公式推导，可以直接参考论文的最后的附录部分，有详细的介绍，在此截图如下： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/WX20180227-004500@2x.png" alt=""> <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/WX20180227-004518@2x.png" alt=""> <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/WX20180227-004547@2x.png" alt=""></p>
                  <h2 id="结语"><a href="#结语" class="headerlink" title="结语"></a>结语</h2>
                  <p>本篇论文算是为 Seq2Seq 的研究开了先河，而且其提出的 GRU 结构也为后来的研究做好了铺垫，得到了广泛应用，非常值得一读。</p>
                  <h2 id="参考"><a href="#参考" class="headerlink" title="参考"></a>参考</h2>
                  <ul>
                    <li><a href="http://www.zmonster.me/notes/phrase_representation_using_rnn_encoder_decoder.html" target="_blank" rel="noopener">http://www.zmonster.me/notes/phrase_representation_using_rnn_encoder_decoder.html</a></li>
                    <li><a href="https://yq.aliyun.com/articles/175467" target="_blank" rel="noopener">https://yq.aliyun.com/articles/175467</a></li>
                  </ul>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-02-27 00:52:07" itemprop="dateCreated datePublished" datetime="2018-02-27T00:52:07+08:00">2018-02-27</time>
                </span>
                <span id="/5737.html" class="post-meta-item leancloud_visitors" data-flag-title="[论文笔记] Learning Phrase Representations using RNN Encoder–Decoder for Statistical Machine Translation" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>1.5k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5715.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5715.html" class="post-title-link" itemprop="url">TensorFlow layers模块用法</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>TensorFlow 中的 layers 模块提供用于深度学习的更高层次封装的 API，利用它我们可以轻松地构建模型，这一节我们就来看下这个模块的 API 的具体用法。</p>
                  <h2 id="概览"><a href="#概览" class="headerlink" title="概览"></a>概览</h2>
                  <p>layers 模块的路径写法为 tf.layers，这个模块定义在 tensorflow/python/layers/layers.py，其官方文档地址为：<a href="https://www.tensorflow.org/api_docs/python/tf/layers" target="_blank" rel="noopener">https://www.tensorflow.org/api_docs/python/tf/layers</a>，TensorFlow 版本为 1.5。 这里面提供了多个类和方法以供使用，下面我们分别予以介绍。</p>
                  <h2 id="方法"><a href="#方法" class="headerlink" title="方法"></a>方法</h2>
                  <p>tf.layers 模块提供的方法有：</p>
                  <ul>
                    <li>Input(…): 用于实例化一个输入 Tensor，作为神经网络的输入。</li>
                    <li>average_pooling1d(…): 一维平均池化层</li>
                    <li>average_pooling2d(…): 二维平均池化层</li>
                    <li>average_pooling3d(…): 三维平均池化层</li>
                    <li>batch_normalization(…): 批量标准化层</li>
                    <li>conv1d(…): 一维卷积层</li>
                    <li>conv2d(…): 二维卷积层</li>
                    <li>conv2d_transpose(…): 二维反卷积层</li>
                    <li>conv3d(…): 三维卷积层</li>
                    <li>conv3d_transpose(…): 三维反卷积层</li>
                    <li>dense(…): 全连接层</li>
                    <li>dropout(…): Dropout层</li>
                    <li>flatten(…): Flatten层，即把一个 Tensor 展平</li>
                    <li>max_pooling1d(…): 一维最大池化层</li>
                    <li>max_pooling2d(…): 二维最大池化层</li>
                    <li>max_pooling3d(…): 三维最大池化层</li>
                    <li>separable_conv2d(…): 二维深度可分离卷积层</li>
                  </ul>
                  <h3 id="Input"><a href="#Input" class="headerlink" title="Input"></a>Input</h3>
                  <p>tf.layers.Input() 这个方法是用于输入数据的方法，其实类似于 tf.placeholder，相当于一个占位符的作用，当然也可以通过传入 tensor 参数来进行赋值。</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Input(</span><br><span class="line">    <span class="attribute">shape</span>=None,</span><br><span class="line">    <span class="attribute">batch_size</span>=None,</span><br><span class="line">    <span class="attribute">name</span>=None,</span><br><span class="line">    <span class="attribute">dtype</span>=tf.float32,</span><br><span class="line">    <span class="attribute">sparse</span>=<span class="literal">False</span>,</span><br><span class="line">    <span class="attribute">tensor</span>=None</span><br><span class="line">)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>参数说明如下：</p>
                  <ul>
                    <li>shape：可选，默认 None，是一个数字组成的元组或列表，但是这个 shape 比较特殊，它不包含 batch_size，比如传入的 shape 为 [32]，那么它会将 shape 转化为 [?, 32]，这里一定需要注意。</li>
                    <li>batch_size：可选，默认 None，代表输入数据的 batch size，可以是数字或者 None。</li>
                    <li>name：可选，默认 None，输入层的名称。</li>
                    <li>dtype：可选，默认 tf.float32，元素的类型。</li>
                    <li>sparse：可选，默认 False，指定是否以稀疏矩阵的形式来创建 placeholder。</li>
                    <li>tensor：可选，默认 None，如果指定，那么创建的内容便不再是一个 placeholder，会用此 Tensor 初始化。</li>
                  </ul>
                  <p>返回值： 返回一个包含历史 Meta Data 的 Tensor。 我们用一个实例来感受一下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">x = tf<span class="selector-class">.layers</span>.Input(shape=[<span class="number">32</span>])</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(x)</span></span></span><br><span class="line">y = tf<span class="selector-class">.layers</span>.dense(x, <span class="number">16</span>, activation=tf<span class="selector-class">.nn</span>.softmax)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(y)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>首先我们用 Input() 方法初始化了一个 placeholder，这时我们没有传入 tensor 参数，然后调用了 dense() 方法构建了一个全连接网络，激活函数使用 softmax，然后将二者输出，结果如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="title">Tensor</span><span class="params">(<span class="string">"input_layer_1:0"</span>, shape=(?, <span class="number">32</span>)</span></span>, dtype=float32)</span><br><span class="line"><span class="function"><span class="title">Tensor</span><span class="params">(<span class="string">"dense/Softmax:0"</span>, shape=(?, <span class="number">16</span>)</span></span>, dtype=float32)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这时我们发现，shape 它给我们做了转化，本来是 [32]，结果它给转化成了 [?, 32]，即第一维代表 batch_size，所以我们需要注意，在调用此方法的时候不需要去关心 batch_size 这一维。 如果我们在初始化的时候传入一个已有 Tensor，例如：</p>
                  <figure class="highlight haskell">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="class"><span class="keyword">data</span> = tf.constant([1, 2, 3])</span></span><br><span class="line"><span class="title">x</span> = tf.layers.<span class="type">Input</span>(tensor=<span class="class"><span class="keyword">data</span>)</span></span><br><span class="line"><span class="title">print</span>(x)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="constructor">Tensor(<span class="string">"Const:0"</span>, <span class="params">shape</span>=(3,)</span>, dtype=<span class="built_in">int32</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到它可以自动计算出其 shape 和 dtype。</p>
                  <h3 id="batch-normalization"><a href="#batch-normalization" class="headerlink" title="batch_normalization"></a>batch_normalization</h3>
                  <p>此方法是批量标准化的方法，经过处理之后可以加速训练速度，其定义在 tensorflow/python/layers/normalization.py，论文可以参考：<a href="http://arxiv.org/abs/1502.03167" target="_blank" rel="noopener">http://arxiv.org/abs/1502.03167</a> “Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift”。</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">batch_normalization(</span><br><span class="line">    inputs,</span><br><span class="line">    <span class="attribute">axis</span>=-1,</span><br><span class="line">    <span class="attribute">momentum</span>=0.99,</span><br><span class="line">    <span class="attribute">epsilon</span>=0.001,</span><br><span class="line">    <span class="attribute">center</span>=<span class="literal">True</span>,</span><br><span class="line">    <span class="attribute">scale</span>=<span class="literal">True</span>,</span><br><span class="line">    <span class="attribute">beta_initializer</span>=tf.zeros_initializer(),</span><br><span class="line">    <span class="attribute">gamma_initializer</span>=tf.ones_initializer(),</span><br><span class="line">    <span class="attribute">moving_mean_initializer</span>=tf.zeros_initializer(),</span><br><span class="line">    <span class="attribute">moving_variance_initializer</span>=tf.ones_initializer(),</span><br><span class="line">    <span class="attribute">beta_regularizer</span>=None,</span><br><span class="line">    <span class="attribute">gamma_regularizer</span>=None,</span><br><span class="line">    <span class="attribute">beta_constraint</span>=None,</span><br><span class="line">    <span class="attribute">gamma_constraint</span>=None,</span><br><span class="line">    <span class="attribute">training</span>=<span class="literal">False</span>,</span><br><span class="line">    <span class="attribute">trainable</span>=<span class="literal">True</span>,</span><br><span class="line">    <span class="attribute">name</span>=None,</span><br><span class="line">    <span class="attribute">reuse</span>=None,</span><br><span class="line">    <span class="attribute">renorm</span>=<span class="literal">False</span>,</span><br><span class="line">    <span class="attribute">renorm_clipping</span>=None,</span><br><span class="line">    <span class="attribute">renorm_momentum</span>=0.99,</span><br><span class="line">    <span class="attribute">fused</span>=None,</span><br><span class="line">    <span class="attribute">virtual_batch_size</span>=None,</span><br><span class="line">    <span class="attribute">adjustment</span>=None</span><br><span class="line">)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>参数说明如下：</p>
                  <ul>
                    <li>inputs：必需，即输入数据。</li>
                    <li>axis：可选，默认 -1，即进行标注化操作时操作数据的哪个维度。</li>
                    <li>momentum：可选，默认 0.99，即动态均值的动量。</li>
                    <li>epsilon：可选，默认 0.01，大于0的小浮点数，用于防止除0错误。</li>
                    <li>center：可选，默认 True，若设为True，将会将 beta 作为偏置加上去，否则忽略参数 beta</li>
                    <li>scale：可选，默认 True，若设为True，则会乘以gamma，否则不使用gamma。当下一层是线性的时，可以设False，因为scaling的操作将被下一层执行。</li>
                    <li>beta_initializer：可选，默认 zeros_initializer，即 beta 权重的初始方法。</li>
                    <li>gamma_initializer：可选，默认 ones_initializer，即 gamma 的初始化方法。</li>
                    <li>moving_mean_initializer：可选，默认 zeros_initializer，即动态均值的初始化方法。</li>
                    <li>moving_variance_initializer：可选，默认 ones_initializer，即动态方差的初始化方法。</li>
                    <li>beta_regularizer: 可选，默认None，beta 的正则化方法。</li>
                    <li>gamma_regularizer: 可选，默认None，gamma 的正则化方法。</li>
                    <li>beta_constraint: 可选，默认None，加在 beta 上的约束项。</li>
                    <li>gamma_constraint: 可选，默认None，加在 gamma 上的约束项。</li>
                    <li>training：可选，默认 False，返回结果是 training 模式。</li>
                    <li>trainable：可选，默认为 True，布尔类型，如果为 True，则将变量添加 GraphKeys.TRAINABLE_VARIABLES 中。</li>
                    <li>name：可选，默认 None，层名称。</li>
                    <li>reuse：可选，默认 None，根据层名判断是否重复利用。</li>
                    <li>renorm：可选，默认 False，是否要用 Batch Renormalization (<a href="https://arxiv.org/abs/1702.03275" target="_blank" rel="noopener">https://arxiv.org/abs/1702.03275</a>)</li>
                    <li>renorm_clipping：可选，默认 None，是否要用 rmax、rmin、dmax 来 scalar Tensor。</li>
                    <li>renorm_momentum，可选，默认 0.99，用来更新动态均值和标准差的 Momentum 值。</li>
                    <li>fused，可选，默认 None，是否使用一个更快的、融合的实现方法。</li>
                    <li>virtual_batch_size，可选，默认 None，是一个 int 数字，指定一个虚拟 batch size。</li>
                    <li>adjustment，可选，默认 None，对标准化后的结果进行适当调整的方法。</li>
                  </ul>
                  <p>最后的一些参数说明不够详尽，更详细的用法参考：<a href="https://www.tensorflow.org/api_docs/python/tf/layers/batch_normalization" target="_blank" rel="noopener">https://www.tensorflow.org/api_docs/python/tf/layers/batch_normalization</a>。 其用法很简单，在输入数据后面加一层 batch_normalization() 即可：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">x</span> = tf.layers.Input(shape=[<span class="number">32</span>])</span><br><span class="line"><span class="attr">x</span> = tf.layers.batch_normalization(x)</span><br><span class="line"><span class="attr">y</span> = tf.layers.dense(x, <span class="number">20</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="dense"><a href="#dense" class="headerlink" title="dense"></a>dense</h3>
                  <p>dense，即全连接网络，layers 模块提供了一个 dense() 方法来实现此操作，定义在 tensorflow/python/layers/core.py 中，下面我们来说明一下它的用法。</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">dense(</span><br><span class="line">    inputs,</span><br><span class="line">    units,</span><br><span class="line">    <span class="attribute">activation</span>=None,</span><br><span class="line">    <span class="attribute">use_bias</span>=<span class="literal">True</span>,</span><br><span class="line">    <span class="attribute">kernel_initializer</span>=None,</span><br><span class="line">    <span class="attribute">bias_initializer</span>=tf.zeros_initializer(),</span><br><span class="line">    <span class="attribute">kernel_regularizer</span>=None,</span><br><span class="line">    <span class="attribute">bias_regularizer</span>=None,</span><br><span class="line">    <span class="attribute">activity_regularizer</span>=None,</span><br><span class="line">    <span class="attribute">kernel_constraint</span>=None,</span><br><span class="line">    <span class="attribute">bias_constraint</span>=None,</span><br><span class="line">    <span class="attribute">trainable</span>=<span class="literal">True</span>,</span><br><span class="line">    <span class="attribute">name</span>=None,</span><br><span class="line">    <span class="attribute">reuse</span>=None</span><br><span class="line">)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>参数说明如下：</p>
                  <ul>
                    <li>inputs：必需，即需要进行操作的输入数据。</li>
                    <li>units：必须，即神经元的数量。</li>
                    <li>activation：可选，默认为 None，如果为 None 则是线性激活。</li>
                    <li>use_bias：可选，默认为 True，是否使用偏置。</li>
                    <li>kernel_initializer：可选，默认为 None，即权重的初始化方法，如果为 None，则使用默认的 Xavier 初始化方法。</li>
                    <li>bias_initializer：可选，默认为零值初始化，即偏置的初始化方法。</li>
                    <li>kernel_regularizer：可选，默认为 None，施加在权重上的正则项。</li>
                    <li>bias_regularizer：可选，默认为 None，施加在偏置上的正则项。</li>
                    <li>activity_regularizer：可选，默认为 None，施加在输出上的正则项。</li>
                    <li>kernel_constraint，可选，默认为 None，施加在权重上的约束项。</li>
                    <li>bias_constraint，可选，默认为 None，施加在偏置上的约束项。</li>
                    <li>trainable：可选，默认为 True，布尔类型，如果为 True，则将变量添加到 GraphKeys.TRAINABLE_VARIABLES 中。</li>
                    <li>name：可选，默认为 None，卷积层的名称。</li>
                    <li>reuse：可选，默认为 None，布尔类型，如果为 True，那么如果 name 相同时，会重复利用。</li>
                  </ul>
                  <p>返回值： 全连接网络处理后的 Tensor。 下面我们用一个实例来感受一下它的用法：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">x = tf<span class="selector-class">.layers</span>.Input(shape=[<span class="number">32</span>])</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(x)</span></span></span><br><span class="line">y1 = tf<span class="selector-class">.layers</span>.dense(x, <span class="number">16</span>, activation=tf<span class="selector-class">.nn</span>.relu)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(y1)</span></span></span><br><span class="line">y2 = tf<span class="selector-class">.layers</span>.dense(y1, <span class="number">5</span>, activation=tf<span class="selector-class">.nn</span>.sigmoid)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(y2)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>首先我们用 Input 定义了 [?, 32] 的输入数据，然后经过第一层全连接网络，此时指定了神经元个数为 16，激活函数为 relu，接着输出结果经过第二层全连接网络，此时指定了神经元个数为 5，激活函数为 sigmoid，最后输出，结果如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="title">Tensor</span><span class="params">(<span class="string">"input_layer_1:0"</span>, shape=(?, <span class="number">32</span>)</span></span>, dtype=float32)</span><br><span class="line"><span class="function"><span class="title">Tensor</span><span class="params">(<span class="string">"dense/Relu:0"</span>, shape=(?, <span class="number">16</span>)</span></span>, dtype=float32)</span><br><span class="line"><span class="function"><span class="title">Tensor</span><span class="params">(<span class="string">"dense_2/Sigmoid:0"</span>, shape=(?, <span class="number">5</span>)</span></span>, dtype=float32)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到输出结果的最后一维度就等于神经元的个数，这是非常容易理解的。</p>
                  <h3 id="convolution"><a href="#convolution" class="headerlink" title="convolution"></a>convolution</h3>
                  <p>convolution，即卷积，这里提供了多个卷积方法，如 conv1d()、conv2d()、conv3d()，分别代表一维、二维、三维卷积，另外还有 conv2d_transpose()、conv3d_transpose()，分别代表二维和三维反卷积，还有 separable_conv2d() 方法代表二维深度可分离卷积。它们定义在 tensorflow/python/layers/convolutional.py 中，其用法都是类似的，在这里以 conv2d() 方法为例进行说明。</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">conv2d(</span><br><span class="line">    inputs,</span><br><span class="line">    filters,</span><br><span class="line">    kernel_size,</span><br><span class="line">    strides=(1, 1),</span><br><span class="line">    <span class="attribute">padding</span>=<span class="string">'valid'</span>,</span><br><span class="line">    <span class="attribute">data_format</span>=<span class="string">'channels_last'</span>,</span><br><span class="line">    dilation_rate=(1, 1),</span><br><span class="line">    <span class="attribute">activation</span>=None,</span><br><span class="line">    <span class="attribute">use_bias</span>=<span class="literal">True</span>,</span><br><span class="line">    <span class="attribute">kernel_initializer</span>=None,</span><br><span class="line">    <span class="attribute">bias_initializer</span>=tf.zeros_initializer(),</span><br><span class="line">    <span class="attribute">kernel_regularizer</span>=None,</span><br><span class="line">    <span class="attribute">bias_regularizer</span>=None,</span><br><span class="line">    <span class="attribute">activity_regularizer</span>=None,</span><br><span class="line">    <span class="attribute">kernel_constraint</span>=None,</span><br><span class="line">    <span class="attribute">bias_constraint</span>=None,</span><br><span class="line">    <span class="attribute">trainable</span>=<span class="literal">True</span>,</span><br><span class="line">    <span class="attribute">name</span>=None,</span><br><span class="line">    <span class="attribute">reuse</span>=None</span><br><span class="line">)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>参数说明如下：</p>
                  <ul>
                    <li>inputs：必需，即需要进行操作的输入数据。</li>
                    <li>filters：必需，是一个数字，代表了输出通道的个数，即 output_channels。</li>
                    <li>kernel_size：必需，卷积核大小，必须是一个数字（高和宽都是此数字）或者长度为 2 的列表（分别代表高、宽）。</li>
                    <li>strides：可选，默认为 (1, 1)，卷积步长，必须是一个数字（高和宽都是此数字）或者长度为 2 的列表（分别代表高、宽）。</li>
                    <li>padding：可选，默认为 valid，padding 的模式，有 valid 和 same 两种，大小写不区分。</li>
                    <li>data_format：可选，默认 channels_last，分为 channels_last 和 channels_first 两种模式，代表了输入数据的维度类型，如果是 channels_last，那么输入数据的 shape 为 (batch, height, width, channels)，如果是 channels_first，那么输入数据的 shape 为 (batch, channels, height, width)。</li>
                    <li>dilation_rate：可选，默认为 (1, 1)，卷积的扩张率，如当扩张率为 2 时，卷积核内部就会有边距，3x3 的卷积核就会变成 5x5。</li>
                    <li>activation：可选，默认为 None，如果为 None 则是线性激活。</li>
                    <li>use_bias：可选，默认为 True，是否使用偏置。</li>
                    <li>kernel_initializer：可选，默认为 None，即权重的初始化方法，如果为 None，则使用默认的 Xavier 初始化方法。</li>
                    <li>bias_initializer：可选，默认为零值初始化，即偏置的初始化方法。</li>
                    <li>kernel_regularizer：可选，默认为 None，施加在权重上的正则项。</li>
                    <li>bias_regularizer：可选，默认为 None，施加在偏置上的正则项。</li>
                    <li>activity_regularizer：可选，默认为 None，施加在输出上的正则项。</li>
                    <li>kernel_constraint，可选，默认为 None，施加在权重上的约束项。</li>
                    <li>bias_constraint，可选，默认为 None，施加在偏置上的约束项。</li>
                    <li>trainable：可选，默认为 True，布尔类型，如果为 True，则将变量添加到 GraphKeys.TRAINABLE_VARIABLES 中。</li>
                    <li>name：可选，默认为 None，卷积层的名称。</li>
                    <li>reuse：可选，默认为 None，布尔类型，如果为 True，那么如果 name 相同时，会重复利用。</li>
                  </ul>
                  <p>返回值： 卷积后的 Tensor。 下面我们用实例感受一下它的用法：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">x = tf.layers.Input(shape=[20, 20, 3])</span><br><span class="line">y = tf.layers.conv2d(x, <span class="attribute">filters</span>=6, <span class="attribute">kernel_size</span>=2, <span class="attribute">padding</span>=<span class="string">'same'</span>)</span><br><span class="line"><span class="builtin-name">print</span>(y)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们首先声明了一个 [?, 20, 20, 3] 的输入 x，然后将其传给 conv2d() 方法，filters 设定为 6，即输出通道为 6，kernel_size 为 2，即卷积核大小为 2 x 2，padding 方式设置为 same，那么输出结果的宽高和原来一定是相同的，但是输出通道就变成了 6，结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Tensor(<span class="string">"conv2d/BiasAdd:0"</span>, shape=(?, <span class="number">20</span>, <span class="number">20</span>, <span class="number">6</span>), dtype=<span class="built_in">float</span>32)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>但如果我们将 padding 方式不传入，使用默认的 valid 模式，代码改写如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">x = tf.layers.Input(shape=[20, 20, 3])</span><br><span class="line">y = tf.layers.conv2d(x, <span class="attribute">filters</span>=6, <span class="attribute">kernel_size</span>=2)</span><br><span class="line"><span class="builtin-name">print</span>(y)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Tensor(<span class="string">"conv2d/BiasAdd:0"</span>, shape=(?, <span class="number">19</span>, <span class="number">19</span>, <span class="number">6</span>), dtype=<span class="built_in">float</span>32)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果就变成了 [?, 19, 19, 6]，这是因为步长默认为 1，卷积核大小为 2 x 2，所以得到的结果的高宽即为 (20 - (2 - 1)) x (20 - (2 - 1)) = 19 x 19。 当然卷积核我们也可以变换大小，传入一个列表形式：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">x = tf.layers.Input(shape=[<span class="number">20</span>, <span class="number">20</span>, <span class="number">3</span>])</span><br><span class="line">y = tf.layers.conv2d(x, filters=<span class="number">6</span>, kernel_size=[<span class="number">2</span>, <span class="number">3</span>])</span><br><span class="line">print(y)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这时我们的卷积核大小变成了 2 x 3，即高为 2，宽为 3，结果就变成了 [?, 19, 18, 6]，这是因为步长默认为 1，卷积核大小为 2 x 2，所以得到的结果的高宽即为 (20 - (2 - 1)) x (20 - (3 - 1)) = 19 x 18。 如果我们将步长也设置一下，也传入列表形式：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">x = tf.layers.Input(shape=[<span class="number">20</span>, <span class="number">20</span>, <span class="number">3</span>])</span><br><span class="line">y = tf.layers.conv2d(x, filters=<span class="number">6</span>, kernel_size=[<span class="number">2</span>, <span class="number">3</span>], strides=[<span class="number">2</span>, <span class="number">2</span>])</span><br><span class="line">print(y)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这时卷积核大小变成了 2 x 3，步长变成了 2 x 2，所以结果的高宽为 ceil(20 - (2- 1)) / 2 x ceil(20 - (3- 1)) / 2 = 10 x 9，得到的结果即为 [?, 10, 9, 6]。 运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Tensor(<span class="string">"conv2d_4/BiasAdd:0"</span>, shape=(?, <span class="number">10</span>, <span class="number">9</span>, <span class="number">6</span>), dtype=<span class="built_in">float</span>32)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外我们还可以传入激活函数，或者禁用 bias 等操作，实例如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">x = tf.layers.Input(shape=[20, 20, 3])</span><br><span class="line">y = tf.layers.conv2d(x, <span class="attribute">filters</span>=6, <span class="attribute">kernel_size</span>=2, <span class="attribute">activation</span>=tf.nn.relu, <span class="attribute">use_bias</span>=<span class="literal">False</span>)</span><br><span class="line"><span class="builtin-name">print</span>(y)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们就将激活函数改成了 relu，同时禁用了 bias，运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Tensor(<span class="string">"conv2d_5/Relu:0"</span>, shape=(?, <span class="number">19</span>, <span class="number">19</span>, <span class="number">6</span>), dtype=<span class="built_in">float</span>32)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外还有反卷积操作，反卷积顾名思义即卷积的反向操作，即输入卷积的结果，得到卷积前的结果，其参数用法是完全一样的，例如：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">x = tf.layers.Input(shape=[20, 20, 3])</span><br><span class="line">y = tf.layers.conv2d_transpose(x, <span class="attribute">filters</span>=6, <span class="attribute">kernel_size</span>=2, <span class="attribute">strides</span>=2)</span><br><span class="line"><span class="builtin-name">print</span>(y)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>例如此处输入的图像高宽为 20 x 20，经过卷积核为 2，步长为 2 的反卷积处理，得到的结果高宽就变为了 40 x 40，结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Tensor(<span class="string">"conv2d_transpose/BiasAdd:0"</span>, shape=(?, <span class="number">40</span>, <span class="number">40</span>, <span class="number">6</span>), dtype=<span class="built_in">float</span>32)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="pooling"><a href="#pooling" class="headerlink" title="pooling"></a>pooling</h3>
                  <p>pooling，即池化，layers 模块提供了多个池化方法，这几个池化方法都是类似的，包括 max_pooling1d()、max_pooling2d()、max_pooling3d()、average_pooling1d()、average_pooling2d()、average_pooling3d()，分别代表一维二维三维最大和平均池化方法，它们都定义在 tensorflow/python/layers/pooling.py 中，这里以 max_pooling2d() 方法为例进行介绍。</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">max_pooling2d(</span><br><span class="line">    inputs,</span><br><span class="line">    pool_size,</span><br><span class="line">    strides,</span><br><span class="line">    <span class="attribute">padding</span>=<span class="string">'valid'</span>,</span><br><span class="line">    <span class="attribute">data_format</span>=<span class="string">'channels_last'</span>,</span><br><span class="line">    <span class="attribute">name</span>=None</span><br><span class="line">)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>参数说明如下：</p>
                  <ul>
                    <li>inputs: 必需，即需要池化的输入对象，必须是 4 维的。</li>
                    <li>pool_size：必需，池化窗口大小，必须是一个数字（高和宽都是此数字）或者长度为 2 的列表（分别代表高、宽）。</li>
                    <li>strides：必需，池化步长，必须是一个数字（高和宽都是此数字）或者长度为 2 的列表（分别代表高、宽）。</li>
                    <li>padding：可选，默认 valid，padding 的方法，valid 或者 same，大小写不区分。</li>
                    <li>data_format：可选，默认 channels_last，分为 channels_last 和 channels_first 两种模式，代表了输入数据的维度类型，如果是 channels_last，那么输入数据的 shape 为 (batch, height, width, channels)，如果是 channels_first，那么输入数据的 shape 为 (batch, channels, height, width)。</li>
                    <li>name：可选，默认 None，池化层的名称。</li>
                  </ul>
                  <p>返回值： 经过池化处理后的 Tensor。 下面我们用一个实例来感受一下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">x = tf.layers.Input(shape=[20, 20, 3])</span><br><span class="line"><span class="builtin-name">print</span>(x)</span><br><span class="line">y = tf.layers.conv2d(x, <span class="attribute">filters</span>=6, <span class="attribute">kernel_size</span>=3, <span class="attribute">padding</span>=<span class="string">'same'</span>)</span><br><span class="line"><span class="builtin-name">print</span>(y)</span><br><span class="line">p = tf.layers.max_pooling2d(y, <span class="attribute">pool_size</span>=2, <span class="attribute">strides</span>=2)</span><br><span class="line"><span class="builtin-name">print</span>(p)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里我们首先指定了输入 x，shape 为 [20, 20, 3]，然后对其进行了卷积计算，然后池化，最后得到池化后的结果。结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Tensor(<span class="string">"input_layer_1:0"</span>, shape=(?, <span class="number">20</span>, <span class="number">20</span>, <span class="number">3</span>), dtype=<span class="built_in">float</span>32)</span><br><span class="line">Tensor(<span class="string">"conv2d/BiasAdd:0"</span>, shape=(?, <span class="number">20</span>, <span class="number">20</span>, <span class="number">6</span>), dtype=<span class="built_in">float</span>32)</span><br><span class="line">Tensor(<span class="string">"max_pooling2d/MaxPool:0"</span>, shape=(?, <span class="number">10</span>, <span class="number">10</span>, <span class="number">6</span>), dtype=<span class="built_in">float</span>32)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到这里池化窗口用的是 2，步长也是 2，所以原本卷积后 shape 为 [?, 20, 20, 6] 的结果就变成了 [?, 10, 10, 6]。</p>
                  <h3 id="dropout"><a href="#dropout" class="headerlink" title="dropout"></a>dropout</h3>
                  <p>dropout 是指在深度学习网络的训练过程中，对于神经网络单元，按照一定的概率将其暂时从网络中丢弃，可以用来防止过拟合，layers 模块中提供了 dropout() 方法来实现这一操作，定义在 tensorflow/python/layers/core.py。下面我们来说明一下它的用法。</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">dropout(</span><br><span class="line">    inputs,</span><br><span class="line">    <span class="attribute">rate</span>=0.5,</span><br><span class="line">    <span class="attribute">noise_shape</span>=None,</span><br><span class="line">    <span class="attribute">seed</span>=None,</span><br><span class="line">    <span class="attribute">training</span>=<span class="literal">False</span>,</span><br><span class="line">    <span class="attribute">name</span>=None</span><br><span class="line">)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>参数说明如下：</p>
                  <ul>
                    <li>inputs：必须，即输入数据。</li>
                    <li>rate：可选，默认为 0.5，即 dropout rate，如设置为 0.1，则意味着会丢弃 10% 的神经元。</li>
                    <li>noise_shape：可选，默认为 None，int32 类型的一维 Tensor，它代表了 dropout mask 的 shape，dropout mask 会与 inputs 相乘对 inputs 做转换，例如 inputs 的 shape 为 (batch_size, timesteps, features)，但我们想要 droput mask 在所有 timesteps 都是相同的，我们可以设置 noise_shape=[batch_size, 1, features]。</li>
                    <li>seed：可选，默认为 None，即产生随机熟的种子值。</li>
                    <li>training：可选，默认为 False，布尔类型，即代表了是否标志位 training 模式。</li>
                    <li>name：可选，默认为 None，dropout 层的名称。</li>
                  </ul>
                  <p>返回： 经过 dropout 层之后的 Tensor。 我们用一个实例来感受一下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">x = tf<span class="selector-class">.layers</span>.Input(shape=[<span class="number">32</span>])</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(x)</span></span></span><br><span class="line">y = tf<span class="selector-class">.layers</span>.dense(x, <span class="number">16</span>, activation=tf<span class="selector-class">.nn</span>.softmax)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(y)</span></span></span><br><span class="line">d = tf<span class="selector-class">.layers</span>.dropout(y, rate=<span class="number">0.2</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(d)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="title">Tensor</span><span class="params">(<span class="string">"input_layer_1:0"</span>, shape=(?, <span class="number">32</span>)</span></span>, dtype=float32)</span><br><span class="line"><span class="function"><span class="title">Tensor</span><span class="params">(<span class="string">"dense/Softmax:0"</span>, shape=(?, <span class="number">16</span>)</span></span>, dtype=float32)</span><br><span class="line"><span class="function"><span class="title">Tensor</span><span class="params">(<span class="string">"dropout/Identity:0"</span>, shape=(?, <span class="number">16</span>)</span></span>, dtype=float32)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里我们使用 dropout() 方法实现了 droput 操作，并制定 dropout rate 为 0.2，最后输出结果的 shape 和原来是一致的。</p>
                  <h3 id="flatten"><a href="#flatten" class="headerlink" title="flatten"></a>flatten</h3>
                  <p>flatten() 方法可以对 Tensor 进行展平操作，定义在 tensorflow/python/layers/core.py。</p>
                  <figure class="highlight fortran">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">flatten(</span><br><span class="line">    inputs,</span><br><span class="line">    <span class="keyword">name</span>=<span class="keyword">None</span></span><br><span class="line">)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>参数说明如下：</p>
                  <ul>
                    <li>inputs：必需，即输入数据。</li>
                    <li>name：可选，默认为 None，即该层的名称。</li>
                  </ul>
                  <p>返回结果： 展平后的 Tensor。 下面我们用一个实例来感受一下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">x = tf<span class="selector-class">.layers</span>.Input(shape=[<span class="number">5</span>, <span class="number">6</span>])</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(x)</span></span></span><br><span class="line">y = tf<span class="selector-class">.layers</span>.flatten(x)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(y)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="title">Tensor</span><span class="params">(<span class="string">"input_layer_1:0"</span>, shape=(?, <span class="number">5</span>, <span class="number">6</span>)</span></span>, dtype=float32)</span><br><span class="line"><span class="function"><span class="title">Tensor</span><span class="params">(<span class="string">"flatten/Reshape:0"</span>, shape=(?, <span class="number">30</span>)</span></span>, dtype=float32)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里输入数据的 shape 为 [?, 5, 6]，经过 flatten 层之后，就会变成 [?, 30]，即将除了第一维的数据维度相乘，对原 Tensor 进行展平。 假如第一维是一个已知的数的话，它依然还是同样的处理，示例如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">x = tf.placeholder(shape=[<span class="number">5</span>, <span class="number">6</span>, <span class="number">2</span>], dtype=tf.float32)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(x)</span></span></span><br><span class="line">y = tf<span class="selector-class">.layers</span>.flatten(x)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(y)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="title">Tensor</span><span class="params">(<span class="string">"Placeholder:0"</span>, shape=(<span class="number">5</span>, <span class="number">6</span>, <span class="number">2</span>)</span></span>, dtype=float32)</span><br><span class="line"><span class="function"><span class="title">Tensor</span><span class="params">(<span class="string">"flatten_2/Reshape:0"</span>, shape=(<span class="number">5</span>, <span class="number">12</span>)</span></span>, dtype=float32)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="类"><a href="#类" class="headerlink" title="类"></a>类</h2>
                  <p>除了如上的方法，其实我们还可以直接使用类来进行操作，实际上看方法的实现就是实例化了其对应的类，下面我们首先说明一下有哪些类可以使用：</p>
                  <ul>
                    <li>class AveragePooling1D: 一维平均池化层类</li>
                    <li>class AveragePooling2D: 二维平均池化层类</li>
                    <li>class AveragePooling3D: 三维平均池化层类</li>
                    <li>class BatchNormalization: 批量标准化层类</li>
                    <li>class Conv1D: 一维卷积层类</li>
                    <li>class Conv2D: 二维卷积层类</li>
                    <li>class Conv2DTranspose: 二维反卷积层类</li>
                    <li>class Conv3D: 三维卷积层类</li>
                    <li>class Conv3DTranspose: 三维反卷积层类</li>
                    <li>class Dense: 全连接层类</li>
                    <li>class Dropout: Dropout 层类</li>
                    <li>class Flatten: Flatten 层类</li>
                    <li>class InputSpec: Input 层类</li>
                    <li>class Layer: 基类、父类</li>
                    <li>class MaxPooling1D: 一维最大池化层类</li>
                    <li>class MaxPooling2D: 二维最大池化层类</li>
                    <li>class MaxPooling3D: 三维最大池化层类</li>
                    <li>class SeparableConv2D: 二维深度可分离卷积层类</li>
                  </ul>
                  <p>其实类这些类都和上文介绍的方法是一一对应的，关于它的用法我们可以在方法的源码实现里面找到，下面我们以 Dense 类的用法为例来说明一下这些类的具体调用方法：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">x = tf.layers.<span class="constructor">Input(<span class="params">shape</span>=[32])</span></span><br><span class="line">dense = tf.layers.<span class="constructor">Dense(16, <span class="params">activation</span>=<span class="params">tf</span>.<span class="params">nn</span>.<span class="params">relu</span>)</span></span><br><span class="line">y = dense.apply(x)</span><br><span class="line">print(y)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们初始化了一个 Dense 类，它只接受一个必须参数，那就是 units，相比 dense() 方法来说它没有了 inputs，因此这个实例化的类和 inputs 是无关的，这样就相当于创建了一个 16 个神经元的全连接层。 但创建了不调用是没有用的，我们要将这个层构建到网络之中，需要调用它的 apply() 方法，而 apply() 方法就接收 inputs 这个参数，返回计算结果，运行结果如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="title">Tensor</span><span class="params">(<span class="string">"dense/Relu:0"</span>, shape=(?, <span class="number">16</span>)</span></span>, dtype=float32)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>因此我们可以发现，这些类在初始化的时候实际上是比其对应的方法少了 inputs 参数，其他的参数都是完全一致的，另外需要调用 apply() 方法才可以应用该层并将其构建到模型中。 所以其他的类的用法在此就不一一赘述了，初始化的参数可以类比其对应的方法，实例化类之后，调用 apply() 方法，可以达到同样的构建模型的效果。</p>
                  <h2 id="结语"><a href="#结语" class="headerlink" title="结语"></a>结语</h2>
                  <p>以上便是 TensorFlow layers 模块的详细用法说明，更加详细的用法可以参考官方文档：<a href="https://www.tensorflow.org/api_docs/python/tf/layers" target="_blank" rel="noopener">https://www.tensorflow.org/api_docs/python/tf/layers</a>。 本节代码地址：<a href="https://github.com/AIDeepLearning/TensorFlowLayers" target="_blank" rel="noopener">https://github.com/AIDeepLearning/TensorFlowLayers</a>。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-02-23 03:42:58" itemprop="dateCreated datePublished" datetime="2018-02-23T03:42:58+08:00">2018-02-23</time>
                </span>
                <span id="/5715.html" class="post-meta-item leancloud_visitors" data-flag-title="TensorFlow layers模块用法" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>13k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>12 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5709.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5709.html" class="post-title-link" itemprop="url">TensorFlow验证码识别</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>本节我们来用 TensorFlow 来实现一个深度学习模型，用来实现验证码识别的过程，这里我们识别的验证码是图形验证码，首先我们会用标注好的数据来训练一个模型，然后再用模型来实现这个验证码的识别。</p>
                  <h2 id="验证码"><a href="#验证码" class="headerlink" title="验证码"></a>验证码</h2>
                  <p>首先我们来看下验证码是怎样的，这里我们使用 Python 的 captcha 库来生成即可，这个库默认是没有安装的，所以这里我们需要先安装这个库，另外我们还需要安装 pillow 库，使用 pip3 即可：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> captcha pillow</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>安装好之后，我们就可以用如下代码来生成一个简单的图形验证码了：</p>
                  <figure class="highlight sqf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> captcha.<span class="built_in">image</span> import ImageCaptcha</span><br><span class="line"><span class="keyword">from</span> PIL import <span class="built_in">Image</span></span><br><span class="line"></span><br><span class="line"><span class="built_in">text</span> = <span class="string">'1234'</span></span><br><span class="line"><span class="built_in">image</span> = ImageCaptcha()</span><br><span class="line">captcha = <span class="built_in">image</span>.generate(<span class="built_in">text</span>)</span><br><span class="line">captcha_image = <span class="built_in">Image</span>.open(captcha)</span><br><span class="line">captcha_image.show()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行之后便会弹出一张图片，结果如下： <img src="https://germey.gitbooks.io/ai/content/assets/2018-02-20-23-03-18.jpg" alt=""> 可以看到图中的文字正是我们所定义的 text 内容，这样我们就可以得到一张图片和其对应的真实文本，这样我们就可以用它来生成一批训练数据和测试数据了。</p>
                  <h2 id="预处理"><a href="#预处理" class="headerlink" title="预处理"></a>预处理</h2>
                  <p>在训练之前肯定是要进行数据预处理了，现在我们首先定义好了要生成的验证码文本内容，这就相当于已经有了 label 了，然后我们再用它来生成验证码，就可以得到输入数据 x 了，在这里我们首先定义好我们的输入词表，由于大小写字母加数字的词表比较庞大，设想我们用含有大小写字母和数字的验证码，一个验证码四个字符，那么一共可能的组合是 (26 + 26 + 10) ^ 4 = 14776336 种组合，这个数量训练起来有点大，所以这里我们精简一下，只使用纯数字的验证码来训练，这样其组合个数就变为 10 ^ 4 = 10000 种，显然少了很多。 所以在这里我们先定义一个词表和其长度变量：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">VOCAB</span> = [<span class="string">'0'</span>, <span class="string">'1'</span>, <span class="string">'2'</span>, <span class="string">'3'</span>, <span class="string">'4'</span>, <span class="string">'5'</span>, <span class="string">'6'</span>, <span class="string">'7'</span>, <span class="string">'8'</span>, <span class="string">'9'</span>]</span><br><span class="line"><span class="attr">CAPTCHA_LENGTH</span> = <span class="number">4</span></span><br><span class="line"><span class="attr">VOCAB_LENGTH</span> = len(VOCAB)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里 VOCAB 就是词表的内容，即 0 到 9 这 10 个数字，验证码的字符个数即 CAPTCHA_LENGTH 是 4，词表长度是 VOCAB 的长度，即 10。 接下来我们定义一个生成验证码数据的方法，流程类似上文，只不过这里我们将返回的数据转为了 Numpy 形式的数组：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> PIL <span class="keyword">import</span> Image</span><br><span class="line"><span class="keyword">from</span> captcha.image <span class="keyword">import</span> ImageCaptcha</span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">generate_captcha</span><span class="params">(captcha_text)</span>:</span></span><br><span class="line">    <span class="string">"""</span></span><br><span class="line"><span class="string">    get captcha text and np array</span></span><br><span class="line"><span class="string">    :param captcha_text: source text</span></span><br><span class="line"><span class="string">    :return: captcha image and array</span></span><br><span class="line"><span class="string">    """</span></span><br><span class="line">    image = ImageCaptcha()</span><br><span class="line">    captcha = image.generate(captcha_text)</span><br><span class="line">    captcha_image = Image.open(captcha)</span><br><span class="line">    captcha_array = np.array(captcha_image)</span><br><span class="line">    <span class="keyword">return</span> captcha_array</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样调用此方法，我们就可以得到一个 Numpy 数组了，这个其实是把验证码转化成了每个像素的 RGB，我们调用一下这个方法试试：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">captcha = generate_captcha(<span class="string">'1234'</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(captcha, captcha.shape)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>内容如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[[[<span class="number">239</span> <span class="number">244</span> <span class="number">244</span>]</span><br><span class="line">  [<span class="number">239</span> <span class="number">244</span> <span class="number">244</span>]</span><br><span class="line">  [<span class="number">239</span> <span class="number">244</span> <span class="number">244</span>]</span><br><span class="line">  ..., </span><br><span class="line">  ..., </span><br><span class="line">  [<span class="number">239</span> <span class="number">244</span> <span class="number">244</span>]</span><br><span class="line">  [<span class="number">239</span> <span class="number">244</span> <span class="number">244</span>]</span><br><span class="line">  [<span class="number">239</span> <span class="number">244</span> <span class="number">244</span>]]] </span><br><span class="line">(<span class="number">60</span>, <span class="number">160</span>, <span class="number">3</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到它的 shape 是 (60, 160, 3)，这其实代表验证码图片的高度是 60，宽度是 160，是 60 x 160 像素的验证码，每个像素都有 RGB 值，所以最后一维即为像素的 RGB 值。 接下来我们需要定义 label，由于我们需要使用深度学习模型进行训练，所以这里我们的 label 数据最好使用 One-Hot 编码，即如果验证码文本是 1234，那么应该词表索引位置置 1，总共的长度是 40，我们用程序实现一下 One-Hot 编码和文本的互相转换：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">text2vec</span><span class="params">(text)</span>:</span></span><br><span class="line">    <span class="string">"""</span></span><br><span class="line"><span class="string">    text to one-hot vector</span></span><br><span class="line"><span class="string">    :param text: source text</span></span><br><span class="line"><span class="string">    :return: np array</span></span><br><span class="line"><span class="string">    """</span></span><br><span class="line">    <span class="keyword">if</span> len(text) &gt; CAPTCHA_LENGTH:</span><br><span class="line">        <span class="keyword">return</span> <span class="literal">False</span></span><br><span class="line">    vector = np.zeros(CAPTCHA_LENGTH * VOCAB_LENGTH)</span><br><span class="line">    <span class="keyword">for</span> i, c <span class="keyword">in</span> enumerate(text):</span><br><span class="line">        index = i * VOCAB_LENGTH + VOCAB.index(c)</span><br><span class="line">        vector[index] = <span class="number">1</span></span><br><span class="line">    <span class="keyword">return</span> vector</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">vec2text</span><span class="params">(vector)</span>:</span></span><br><span class="line">    <span class="string">"""</span></span><br><span class="line"><span class="string">    vector to captcha text</span></span><br><span class="line"><span class="string">    :param vector: np array</span></span><br><span class="line"><span class="string">    :return: text</span></span><br><span class="line"><span class="string">    """</span></span><br><span class="line">    <span class="keyword">if</span> <span class="keyword">not</span> isinstance(vector, np.ndarray):</span><br><span class="line">        vector = np.asarray(vector)</span><br><span class="line">    vector = np.reshape(vector, [CAPTCHA_LENGTH, <span class="number">-1</span>])</span><br><span class="line">    text = <span class="string">''</span></span><br><span class="line">    <span class="keyword">for</span> item <span class="keyword">in</span> vector:</span><br><span class="line">        text += VOCAB[np.argmax(item)]</span><br><span class="line">    <span class="keyword">return</span> text</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里 text2vec() 方法就是将真实文本转化为 One-Hot 编码，vec2text() 方法就是将 One-Hot 编码转回真实文本。 例如这里调用一下这两个方法，我们将 1234 文本转换为 One-Hot 编码，然后在将其转回来：</p>
                  <figure class="highlight mel">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">vector</span> = text2vec(<span class="string">'1234'</span>)</span><br><span class="line"><span class="keyword">text</span> = vec2text(<span class="keyword">vector</span>)</span><br><span class="line"><span class="keyword">print</span>(<span class="keyword">vector</span>, <span class="keyword">text</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[ <span class="number">0.</span>  <span class="number">1.</span>  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">1.</span>  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">0.</span></span><br><span class="line">  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">1.</span>  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">1.</span>  <span class="number">0.</span></span><br><span class="line">  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">0.</span>  <span class="number">0.</span>]</span><br><span class="line"><span class="number">1234</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们就可以实现文本到 One-Hot 编码的互转了。 接下来我们就可以构造一批数据了，x 数据就是验证码的 Numpy 数组，y 数据就是验证码的文本的 One-Hot 编码，生成内容如下：</p>
                  <figure class="highlight processing">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> <span class="built_in">random</span></span><br><span class="line">from os.path <span class="keyword">import</span> <span class="built_in">join</span>, exists</span><br><span class="line"><span class="keyword">import</span> pickle</span><br><span class="line"><span class="keyword">import</span> numpy as np</span><br><span class="line">from os <span class="keyword">import</span> makedirs</span><br><span class="line"></span><br><span class="line">DATA_LENGTH = <span class="number">10000</span></span><br><span class="line">DATA_PATH = <span class="string">'data'</span></span><br><span class="line"></span><br><span class="line">def get_random_text():</span><br><span class="line">    <span class="built_in">text</span> = <span class="string">''</span></span><br><span class="line">    <span class="keyword">for</span> i in range(CAPTCHA_LENGTH):</span><br><span class="line">        <span class="built_in">text</span> += <span class="built_in">random</span>.choice(VOCAB)</span><br><span class="line">    <span class="keyword">return</span> <span class="built_in">text</span></span><br><span class="line"></span><br><span class="line">def generate_data():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">'Generating Data...'</span>)</span><br><span class="line">    data_x, data_y = [], []</span><br><span class="line"></span><br><span class="line">    # generate data x and y</span><br><span class="line">    <span class="keyword">for</span> i in range(DATA_LENGTH):</span><br><span class="line">        <span class="built_in">text</span> = get_random_text()</span><br><span class="line">        # <span class="built_in">get</span> captcha array</span><br><span class="line">        captcha_array = generate_captcha(<span class="built_in">text</span>)</span><br><span class="line">        # <span class="built_in">get</span> vector</span><br><span class="line">        vector = text2vec(<span class="built_in">text</span>)</span><br><span class="line">        data_x.<span class="built_in">append</span>(captcha_array)</span><br><span class="line">        data_y.<span class="built_in">append</span>(vector)</span><br><span class="line"></span><br><span class="line">    # write data to pickle</span><br><span class="line">    <span class="keyword">if</span> not exists(DATA_PATH):</span><br><span class="line">        makedirs(DATA_PATH)</span><br><span class="line"></span><br><span class="line">    x = np.asarray(data_x, np.float32)</span><br><span class="line">    y = np.asarray(data_y, np.float32)</span><br><span class="line">    with <span class="built_in">open</span>(<span class="built_in">join</span>(DATA_PATH, <span class="string">'data.pkl'</span>), <span class="string">'wb'</span>) as f:</span><br><span class="line">        pickle.dump(x, f)</span><br><span class="line">        pickle.dump(y, f)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们定义了一个 get_random_text() 方法，可以随机生成验证码文本，然后接下来再利用这个随机生成的文本来产生对应的 x、y 数据，然后我们再将数据写入到 pickle 文件里，这样就完成了预处理的操作。</p>
                  <h2 id="构建模型"><a href="#构建模型" class="headerlink" title="构建模型"></a>构建模型</h2>
                  <p>有了数据之后，我们就开始构建模型吧，这里我们还是利用 train_test_split() 方法将数据分为三部分，训练集、开发集、验证集：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">with</span> <span class="keyword">open</span>('data.pkl', 'rb') <span class="keyword">as</span> f:</span><br><span class="line">    data_x = pickle.load(f)</span><br><span class="line">    data_y = pickle.load(f)</span><br><span class="line">    return standardize(data_x), data_y</span><br><span class="line"></span><br><span class="line">train_x, test_x, train_y, test_y = train<span class="constructor">_test_split(<span class="params">data_x</span>, <span class="params">data_y</span>, <span class="params">test_size</span>=0.4, <span class="params">random_state</span>=40)</span></span><br><span class="line">dev_x, test_x, dev_y, test_y, = train<span class="constructor">_test_split(<span class="params">test_x</span>, <span class="params">test_y</span>, <span class="params">test_size</span>=0.5, <span class="params">random_state</span>=40)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来我们使用者三个数据集构建三个 Dataset 对象：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"># train <span class="keyword">and</span> dev dataset</span><br><span class="line">train_dataset = tf.data.<span class="module-access"><span class="module"><span class="identifier">Dataset</span>.</span></span>from<span class="constructor">_tensor_slices((<span class="params">train_x</span>, <span class="params">train_y</span>)</span>).shuffle(<span class="number">10000</span>)</span><br><span class="line">train_dataset = train_dataset.batch(<span class="module-access"><span class="module"><span class="identifier">FLAGS</span>.</span></span>train_batch_size)</span><br><span class="line"></span><br><span class="line">dev_dataset = tf.data.<span class="module-access"><span class="module"><span class="identifier">Dataset</span>.</span></span>from<span class="constructor">_tensor_slices((<span class="params">dev_x</span>, <span class="params">dev_y</span>)</span>)</span><br><span class="line">dev_dataset = dev_dataset.batch(<span class="module-access"><span class="module"><span class="identifier">FLAGS</span>.</span></span>dev_batch_size)</span><br><span class="line"></span><br><span class="line">test_dataset = tf.data.<span class="module-access"><span class="module"><span class="identifier">Dataset</span>.</span></span>from<span class="constructor">_tensor_slices((<span class="params">test_x</span>, <span class="params">test_y</span>)</span>)</span><br><span class="line">test_dataset = test_dataset.batch(<span class="module-access"><span class="module"><span class="identifier">FLAGS</span>.</span></span>test_batch_size)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后初始化一个迭代器，并绑定到这个数据集上：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"># a reinitializable iterator</span><br><span class="line">iterator = tf.data.<span class="module-access"><span class="module"><span class="identifier">Iterator</span>.</span></span>from<span class="constructor">_structure(<span class="params">train_dataset</span>.<span class="params">output_types</span>, <span class="params">train_dataset</span>.<span class="params">output_shapes</span>)</span></span><br><span class="line">train_initializer = iterator.make<span class="constructor">_initializer(<span class="params">train_dataset</span>)</span></span><br><span class="line">dev_initializer = iterator.make<span class="constructor">_initializer(<span class="params">dev_dataset</span>)</span></span><br><span class="line">test_initializer = iterator.make<span class="constructor">_initializer(<span class="params">test_dataset</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来就是关键的部分了，在这里我们使用三层卷积和两层全连接网络进行构造，在这里为了简化写法，直接使用 TensorFlow 的 layers 模块：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># input Layer</span></span><br><span class="line">with tf.variable_scope(<span class="string">'inputs'</span>):</span><br><span class="line">    # x.shape = [-1, 60, 160, 3]</span><br><span class="line">    x, y_label = iterator.get_next()</span><br><span class="line">keep_prob = tf.placeholder(tf.float32, [])</span><br><span class="line">y = tf.cast(x, tf.float32)</span><br><span class="line"><span class="comment"># 3 CNN layers</span></span><br><span class="line"><span class="keyword">for</span> _ <span class="keyword">in</span> range(3):</span><br><span class="line">    y = tf.layers.conv2d(y, <span class="attribute">filters</span>=32, <span class="attribute">kernel_size</span>=3, <span class="attribute">padding</span>=<span class="string">'same'</span>, <span class="attribute">activation</span>=tf.nn.relu)</span><br><span class="line">    y = tf.layers.max_pooling2d(y, <span class="attribute">pool_size</span>=2, <span class="attribute">strides</span>=2, <span class="attribute">padding</span>=<span class="string">'same'</span>)</span><br><span class="line">    # y = tf.layers.dropout(y, <span class="attribute">rate</span>=keep_prob)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 2 dense layers</span></span><br><span class="line">y = tf.layers.flatten(y)</span><br><span class="line">y = tf.layers.dense(y, 1024, <span class="attribute">activation</span>=tf.nn.relu)</span><br><span class="line">y = tf.layers.dropout(y, <span class="attribute">rate</span>=keep_prob)</span><br><span class="line">y = tf.layers.dense(y, VOCAB_LENGTH)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里卷积核大小为 3，padding 使用 SAME 模式，激活函数使用 relu。 经过全连接网络变换之后，y 的 shape 就变成了 [batch_size, n_classes]，我们的 label 是 CAPTCHA_LENGTH 个 One-Hot 向量拼合而成的，在这里我们想使用交叉熵来计算，但是交叉熵计算的时候，label 参数向量最后一维各个元素之和必须为 1，不然计算梯度的时候会出现问题。详情参见 TensorFlow 的官方文档：<a href="https://www.tensorflow.org/api_docs/python/tf/nn/softmax_cross_entropy_with_logits" target="_blank" rel="noopener">https://www.tensorflow.org/api_docs/python/tf/nn/softmax_cross_entropy_with_logits</a>：</p>
                  <blockquote>
                    <p>NOTE: While the classes are mutually exclusive, their probabilities need not be. All that is required is that each row of labels is a valid probability distribution. If they are not, the computation of the gradient will be incorrect.</p>
                  </blockquote>
                  <p>但是现在的 label 参数是 CAPTCHA_LENGTH 个 One-Hot 向量拼合而成，所以这里各个元素之和为 CAPTCHA_LENGTH，所以我们需要重新 reshape 一下，确保最后一维各个元素之和为 1：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">y_reshape</span> = tf.reshape(y, [-<span class="number">1</span>, VOCAB_LENGTH])</span><br><span class="line"><span class="attr">y_label_reshape</span> = tf.reshape(y_label, [-<span class="number">1</span>, VOCAB_LENGTH])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们就可以确保最后一维是 VOCAB_LENGTH 长度，而它就是一个 One-Hot 向量，所以各元素之和必定为 1。 然后 Loss 和 Accuracy 就好计算了：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"># loss</span><br><span class="line">cross_entropy = tf.reduce<span class="constructor">_sum(<span class="params">tf</span>.<span class="params">nn</span>.<span class="params">softmax_cross_entropy_with_logits</span>(<span class="params">logits</span>=<span class="params">y_reshape</span>, <span class="params">labels</span>=<span class="params">y_label_reshape</span>)</span>)</span><br><span class="line"># accuracy</span><br><span class="line">max_index_predict = tf.argmax(y_reshape, axis=-<span class="number">1</span>)</span><br><span class="line">max_index_label = tf.argmax(y_label_reshape, axis=-<span class="number">1</span>)</span><br><span class="line">correct_predict = tf.equal(max_index_predict, max_index_label)</span><br><span class="line">accuracy = tf.reduce<span class="constructor">_mean(<span class="params">tf</span>.<span class="params">cast</span>(<span class="params">correct_predict</span>, <span class="params">tf</span>.<span class="params">float32</span>)</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>再接下来执行训练即可：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># train</span></span><br><span class="line">train_op = tf.train.RMSPropOptimizer(FLAGS.learning_rate).minimize(cross_entropy, <span class="attribute">global_step</span>=global_step)</span><br><span class="line"><span class="keyword">for</span> epoch <span class="keyword">in</span> range(FLAGS.epoch_num):</span><br><span class="line">    tf.train.global_step(sess, <span class="attribute">global_step_tensor</span>=global_step)</span><br><span class="line">    # train</span><br><span class="line">    sess.<span class="builtin-name">run</span>(train_initializer)</span><br><span class="line">    <span class="keyword">for</span> <span class="keyword">step</span> <span class="keyword">in</span> range(int(train_steps)):</span><br><span class="line">        loss, acc, gstep, _ = sess.<span class="builtin-name">run</span>([cross_entropy, accuracy, global_step, train_op],</span><br><span class="line">                                       feed_dict=&#123;keep_prob: FLAGS.keep_prob&#125;)</span><br><span class="line">        # <span class="builtin-name">print</span> log</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">step</span> % FLAGS.steps_per_print == 0:</span><br><span class="line">            <span class="builtin-name">print</span>(<span class="string">'Global Step'</span>, gstep, <span class="string">'Step'</span>, <span class="keyword">step</span>, <span class="string">'Train Loss'</span>, loss, <span class="string">'Accuracy'</span>, acc)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> epoch % FLAGS.epochs_per_dev == 0:</span><br><span class="line">        # dev</span><br><span class="line">        sess.<span class="builtin-name">run</span>(dev_initializer)</span><br><span class="line">        <span class="keyword">for</span> <span class="keyword">step</span> <span class="keyword">in</span> range(int(dev_steps)):</span><br><span class="line">            <span class="keyword">if</span> <span class="keyword">step</span> % FLAGS.steps_per_print == 0:</span><br><span class="line">                <span class="builtin-name">print</span>(<span class="string">'Dev Accuracy'</span>, sess.<span class="builtin-name">run</span>(accuracy, feed_dict=&#123;keep_prob: 1&#125;), <span class="string">'Step'</span>, <span class="keyword">step</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在这里我们首先初始化 train_initializer，将 iterator 绑定到 Train Dataset 上，然后执行 train_op，获得 loss、acc、gstep 等结果并输出。</p>
                  <h2 id="训练"><a href="#训练" class="headerlink" title="训练"></a>训练</h2>
                  <p>运行训练过程，结果类似如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">...</span><br><span class="line">Dev Accuracy <span class="number">0.9580078</span> Step <span class="number">0</span></span><br><span class="line">Dev Accuracy <span class="number">0.9472656</span> Step <span class="number">2</span></span><br><span class="line">Dev Accuracy <span class="number">0.9501953</span> Step <span class="number">4</span></span><br><span class="line">Dev Accuracy <span class="number">0.9658203</span> Step <span class="number">6</span></span><br><span class="line">Global Step <span class="number">3243</span> Step <span class="number">0</span> Train Loss <span class="number">1.1920928e-06</span> Accuracy <span class="number">1.0</span></span><br><span class="line">Global Step <span class="number">3245</span> Step <span class="number">2</span> Train Loss <span class="number">1.5497207e-06</span> Accuracy <span class="number">1.0</span></span><br><span class="line">Global Step <span class="number">3247</span> Step <span class="number">4</span> Train Loss <span class="number">1.1920928e-06</span> Accuracy <span class="number">1.0</span></span><br><span class="line">Global Step <span class="number">3249</span> Step <span class="number">6</span> Train Loss <span class="number">1.7881392e-06</span> Accuracy <span class="number">1.0</span></span><br><span class="line">...</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>验证集准确率 95% 以上。</p>
                  <h2 id="测试"><a href="#测试" class="headerlink" title="测试"></a>测试</h2>
                  <p>训练过程我们还可以每隔几个 Epoch 保存一下模型：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"># save model</span><br><span class="line"><span class="keyword">if</span> epoch % <span class="module-access"><span class="module"><span class="identifier">FLAGS</span>.</span></span>epochs_per_save<span class="operator"> == </span><span class="number">0</span>:</span><br><span class="line">    saver.save(sess, <span class="module-access"><span class="module"><span class="identifier">FLAGS</span>.</span></span>checkpoint_dir, global_step=gstep)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>当然也可以取验证集上准确率最高的模型进行保存。 验证时我们可以重新 Reload 一下模型，然后进行验证：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># load model</span></span><br><span class="line">ckpt = tf.train.get_checkpoint_state(<span class="string">'ckpt'</span>)</span><br><span class="line"><span class="keyword">if</span> ckpt:</span><br><span class="line">    saver.restore(sess, ckpt.model_checkpoint_path)</span><br><span class="line">    <span class="builtin-name">print</span>(<span class="string">'Restore from'</span>, ckpt.model_checkpoint_path)</span><br><span class="line">    sess.<span class="builtin-name">run</span>(test_initializer)</span><br><span class="line">    <span class="keyword">for</span> <span class="keyword">step</span> <span class="keyword">in</span> range(int(test_steps)):</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">step</span> % FLAGS.steps_per_print == 0:</span><br><span class="line">            <span class="builtin-name">print</span>(<span class="string">'Test Accuracy'</span>, sess.<span class="builtin-name">run</span>(accuracy, feed_dict=&#123;keep_prob: 1&#125;), <span class="string">'Step'</span>, <span class="keyword">step</span>)</span><br><span class="line"><span class="keyword">else</span>:</span><br><span class="line">    <span class="builtin-name">print</span>(<span class="string">'No Model Found'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>验证之后其准确率基本是差不多的。 如果要进行新的 Inference 的话，可以替换下 test_x 即可。</p>
                  <h2 id="结语"><a href="#结语" class="headerlink" title="结语"></a>结语</h2>
                  <p>以上便是使用 TensorFlow 进行验证码识别的过程，代码见：<a href="https://github.com/AIDeepLearning/CrackCaptcha" target="_blank" rel="noopener">https://github.com/AIDeepLearning/CrackCaptcha</a>。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-02-21 15:05:35" itemprop="dateCreated datePublished" datetime="2018-02-21T15:05:35+08:00">2018-02-21</time>
                </span>
                <span id="/5709.html" class="post-meta-item leancloud_visitors" data-flag-title="TensorFlow验证码识别" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>8.2k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>7 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5696.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5696.html" class="post-title-link" itemprop="url">利用python库twilio来免费发送短信</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="大家好，我是四毛，最近开通了个人公众号“用Python来编程”，欢迎大家“关注”，这样您就可以收到优质的文章了。"><a href="#大家好，我是四毛，最近开通了个人公众号“用Python来编程”，欢迎大家“关注”，这样您就可以收到优质的文章了。" class="headerlink" title="大家好，我是四毛，最近开通了个人公众号“用Python来编程”，欢迎大家“关注”，这样您就可以收到优质的文章了。"></a>大家好，我是四毛，最近开通了个人公众号<strong>“用Python来编程”</strong>，欢迎大家“<strong>关注”</strong>，这样您就可以收到优质的文章了。</h2>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/201802111155445124jhQyer.yasuotu.gif" alt=""></p>
                  <pre><code>         今天跟大家分享的主题是利用python库twilio来免费发送短信。

         先放一张成品图
</code></pre>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/webwxgetmsgimg-10-180x320.png" alt=""></p>
                  <pre><code>    代码放在了本文最后的地址中，欢迎有需要的自取，有任何也可以在评论或者后台直接私聊我。
</code></pre>
                  <p><strong>正文</strong></p>
                  <pre><code>    眼尖的小伙伴已经发现了上面的短信的前缀显示这个短信来自于一个叫Twilio的免费的账户，今天我们用到的库就是twilio，既然是免费的账户，那么肯定是有一些限制的，这个会在后面提到。
</code></pre>
                  <p>另外要注意的是这个网站从国内访问的时候，可能会因为一些你懂得原因没法访问，那就只好学习一下怎么科学上网了。</p>
                  <p><strong>1.Twilio</strong></p>
                  <pre><code>    Twilio是一个做成开放插件的电话跟踪服务（call-tracking service）。美国当地时间2016年6月23日，云通讯公司Twilio在纽约证券交易所上市（来自于百度百科）
</code></pre>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/Selection_064-320x237.png" alt=""></p>
                  <p><strong>2. 安装</strong></p>
                  <pre><code>    官方文档地址：https://www.twilio.com/docs/libraries/python

    同时官方还提供对以下语言的支持
</code></pre>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/Selection_071.png" alt=""></p>
                  <pre><code>    可以看到，还是很丰富的。

    最简单的方式就是通过pip，执行如下命令：
</code></pre>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip <span class="keyword">install</span> twilio</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p><strong>3.注册账号</strong></p>
                  <pre><code>    安装好库以后，就需要到官方的网页上进行注册了。

    进入官网：https://www.twilio.com
</code></pre>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/Selection_065.png" alt=""></p>
                  <pre><code>    然后进入注册页面
</code></pre>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/Selection_066.png" alt=""></p>
                  <pre><code>    接着通过了人机认证以后，就会对你的手机号码进行认证，这个就不发图片了。
</code></pre>
                  <p><strong>4.</strong> <strong>进入console</strong></p>
                  <pre><code>    注册好了以后，就可以进入我们自己的面板了
</code></pre>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/Selection_067.png" alt=""></p>
                  <pre><code>    图中箭头所指的两个参数是我们代码中需要的， 可以把两个都复制一下；

    既然是发短信，那么肯定是有一个接收者和一个发送者，发送者的号码可不是我们自己刚刚填的号码，而且twilio给我们分配的一个号码，因为我也是前段时间搞好了，所以不太记得这个号码是不是一开始进去就有的了，如果没有的话，那么就点击Get Stared。

    现在我们点击Manage Numbers
</code></pre>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/Selection_068.png" alt=""></p>
                  <pre><code>    这个时候就可以看到我们的号码了，这是重点，记下来
</code></pre>
                  <p><strong>5. 写代码</strong></p>
                  <pre><code>    根据文档的内容，我们编写了下面的代码：
</code></pre>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment">#!/usr/bin/env python</span></span><br><span class="line"><span class="comment"># -*- coding: utf-8 -*-</span></span><br><span class="line"><span class="comment"># @Author  : ShiMeng</span></span><br><span class="line"><span class="comment"># @File    : send_sms.py</span></span><br><span class="line"><span class="comment"># @Software: PyCharm</span></span><br><span class="line"><span class="keyword">from</span> twilio.rest import Client</span><br><span class="line"><span class="comment"># Your Account SID from twilio.com/console</span></span><br><span class="line">account_sid = <span class="string">"your account sid"</span></span><br><span class="line"><span class="comment"># Your Auth Token from twilio.com/console</span></span><br><span class="line">auth_token  = <span class="string">"your token"</span></span><br><span class="line">client = Client(account_sid, auth_token)</span><br><span class="line">message = client.messages.create(</span><br><span class="line"><span class="comment"># 这里中国的号码前面需要加86</span></span><br><span class="line"><span class="attribute">to</span>=<span class="string">"+接收者的号码"</span>,</span><br><span class="line"><span class="attribute">from_</span>=<span class="string">"+twilio给你的号码 "</span>,</span><br><span class="line"><span class="attribute">body</span>=<span class="string">"Hello from Python!"</span>)</span><br><span class="line"><span class="builtin-name">print</span>(message.sid)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <pre><code>    然后执行程序，你应该会碰到下面的错误
</code></pre>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/Selection_069.png" alt=""></p>
                  <pre><code>    可以从报错信息中明显的看到，提示我们说这个号码没有验证，我们可以到验证的网址上验证一下，也可以购买一个高级别的账号来给未验证的号码发送信息。

    而这个就是我一开始提到的免费账号的限制，在这个限制下面如果你想发送信息给一个接收者，这个接收者的号码必须通过验证，语音验证或者短信验证都可以。如果你是想大批量的发那种垃圾信息，那么你不用往下面看了。下面我们就来对号码进行验证。
</code></pre>
                  <p><strong>6. 验证号码</strong></p>
                  <pre><code>    验证网址：https://www.twilio.com/console/phone-numbers/verified
</code></pre>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/Selection_070.png" alt=""></p>
                  <p><strong>7.重新执行代码</strong></p>
                  <pre><code>    这个时候重新执行我们的代码，没有报错的话，接收者就应该收到你的消息了，就像我一开始放的成品图一样。
</code></pre>
                  <p>但是，在我们发送的信息前面，有一段前缀，我查了一下官方的文档，说这个免费的账户，这个前缀是去不掉的。。。。。。</p>
                  <p><strong>8.查看用量</strong></p>
                  <pre><code>    在面板中，点击Usage即可看到我们的用量， 如下图所示
</code></pre>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/Selection_073.png" alt=""></p>
                  <pre><code>    可以看到我们的用量以及花费，这个花费是不需要我们真正的付钱的，官方的解释是：
</code></pre>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/Selection_072.png" alt=""></p>
                  <p><strong>9.打电话</strong></p>
                  <pre><code>    打电话的代码也很简单
</code></pre>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># Download the Python helper library from twilio.com/docs/python/install</span></span><br><span class="line"><span class="keyword">from</span> twilio.rest import Client</span><br><span class="line"><span class="comment"># Your Account Sid and Auth Token from twilio.com/user/account</span></span><br><span class="line">account_sid = <span class="string">"AC8a9ba33072b6a05f2b81126e3e6609b7"</span></span><br><span class="line">auth_token = <span class="string">"f0150d603c1886d93b9d45ff15d84f24"</span></span><br><span class="line">client = Client(account_sid, auth_token)</span><br><span class="line">call = client.calls.create(</span><br><span class="line"><span class="attribute">to</span>=<span class="string">"+接收者号码"</span>,</span><br><span class="line"><span class="attribute">from_</span>=<span class="string">"+你的twilio号码"</span>,</span><br><span class="line"><span class="attribute">url</span>=<span class="string">"http://demo.twilio.com/docs/voice.xml"</span>,</span><br><span class="line"><span class="attribute">method</span>=<span class="string">"GET"</span>,</span><br><span class="line"><span class="attribute">status_callback</span>=<span class="string">"https://www.myapp.com/events"</span>,</span><br><span class="line"><span class="attribute">status_callback_method</span>=<span class="string">"POST"</span>,</span><br><span class="line">status_callback_event=[<span class="string">"initiated"</span>, <span class="string">"ringing"</span>, <span class="string">"answered"</span>, <span class="string">"completed"</span>]</span><br><span class="line">)</span><br><span class="line"><span class="builtin-name">print</span>(call.sid)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <pre><code>    执行程序后，电话也可以接通，但是里面的人会提示你升级账号。。。。。
</code></pre>
                  <p><strong>总结</strong></p>
                  <pre><code>    好了，到这里我们就可以免费的发送短信了。

    通过这个库，我们可以：

    （1）对线上或者线下后台跑的程序进行监控，并及时发送短信报警

    （2）结合树莓派玩一下，可以实现对超多场景的监测

    代码被放在了这里：https://github.com/xiaosimao/wx_code/tree/master/send_sms
</code></pre>
                  <p> <strong>有问题的可以在评论中指出，或者直接在后台发消息给我。</strong></p>
                  <p> <strong>欢迎大家关注我。</strong></p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/四毛" class="author" itemprop="url" rel="index">四毛</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-02-12 16:25:53" itemprop="dateCreated datePublished" datetime="2018-02-12T16:25:53+08:00">2018-02-12</time>
                </span>
                <span id="/5696.html" class="post-meta-item leancloud_visitors" data-flag-title="利用python库twilio来免费发送短信" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>2.5k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>2 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5678.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5678.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 后续章节</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>本书此部分内容属进阶内容，暂不开放。 如需查看更多可以购买书籍查看。 购买地址： <a href="https://item.jd.com/26114674847.html" target="_blank" rel="noopener">https://item.jd.com/26114674847.html</a> <a href="https://item.jd.com/26124473455.html" target="_blank" rel="noopener">https://item.jd.com/26124473455.html</a> 本书由图灵教育-人民邮电出版社出版发行。 作者：崔庆才 视频学习资源：</p>
                  <ul>
                    <li><a href="https://edu.hellobi.com/course/157" target="_blank" rel="noopener">自己动手，丰衣足食！Python3网络爬虫实战案例</a></li>
                    <li><a href="https://edu.hellobi.com/course/156" target="_blank" rel="noopener">Python3爬虫三大案例实战分享</a></li>
                  </ul>
                  <p>全书预览图： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/Python-3%E7%BD%91%E6%A0%BC%E7%88%AC%E8%99%AB%E5%BC%80%E5%8F%91%E5%AE%9E%E6%88%98-%E7%AB%8B%E4%BD%93%E5%9B%BE-857x1100.jpg" alt=""></p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-02-07 11:34:30" itemprop="dateCreated datePublished" datetime="2018-02-07T11:34:30+08:00">2018-02-07</time>
                </span>
                <span id="/5678.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 后续章节" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>191</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5657.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5657.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 7.4-使用Selenium爬取淘宝商品</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>在前一章中，我们已经成功尝试分析Ajax来抓取相关数据，但是并不是所有页面都可以通过分析Ajax来完成抓取。比如，淘宝，它的整个页面数据确实也是通过Ajax获取的，但是这些Ajax接口参数比较复杂，可能会包含加密密钥等，所以如果想自己构造Ajax参数，还是比较困难的。对于这种页面，最方便快捷的抓取方法就是通过Selenium。本节中，我们就用Selenium来模拟浏览器操作，抓取淘宝的商品信息，并将结果保存到MongoDB。</p>
                  <h2 id="1-本节目标"><a href="#1-本节目标" class="headerlink" title="1. 本节目标"></a>1. 本节目标</h2>
                  <p>本节中，我们要利用Selenium抓取淘宝商品并用pyquery解析得到商品的图片、名称、价格、购买人数、店铺名称和店铺所在地信息，并将其保存到MongoDB。</p>
                  <h2 id="2-准备工作"><a href="#2-准备工作" class="headerlink" title="2. 准备工作"></a>2. 准备工作</h2>
                  <p>本节中，我们首先以Chrome为例来讲解Selenium的用法。在开始之前，请确保已经正确安装好Chrome浏览器并配置好了ChromeDriver；另外，还需要正确安装Python的Selenium库；最后，还对接了PhantomJS和Firefox，请确保安装好PhantomJS和Firefox并配置好了GeckoDriver。如果环境没有配置好，可参考第1章。</p>
                  <h2 id="3-接口分析"><a href="#3-接口分析" class="headerlink" title="3. 接口分析"></a>3. 接口分析</h2>
                  <p>首先，我们来看下淘宝的接口，看看它比一般Ajax多了怎样的内容。</p>
                  <p>打开淘宝页面，搜索商品，比如iPad，此时打开开发者工具，截获Ajax请求，我们可以发现获取商品列表的接口，如图7-19所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-19.png" alt="">图7-19 列表接口</p>
                  <p>它的链接包含了几个GET参数，如果要想构造Ajax链接，直接请求再好不过了，它的返回内容是JSON格式，如图7-20所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-20.jpg" alt="">图7-20 JSON数据</p>
                  <p>但是这个Ajax接口包含几个参数，其中<code>_ksTS</code>、<code>rn</code>参数不能直接发现其规律，如果要去探寻它的生成规律，也不是做不到，但这样相对会比较烦琐，所以如果直接用Selenium来模拟浏览器的话，就不需要再关注这些接口参数了，只要在浏览器里面可以看到的，都可以爬取。这也是我们选用Selenium爬取淘宝的原因。</p>
                  <h2 id="4-页面分析"><a href="#4-页面分析" class="headerlink" title="4. 页面分析"></a>4. 页面分析</h2>
                  <p>本节的目标是爬取商品信息。图7-21是一个商品条目，其中包含商品的基本信息，包括商品图片、名称、价格、购买人数、店铺名称和店铺所在地，我们要做的就是将这些信息都抓取下来。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-21.jpg" alt="">图7-21 商品条目</p>
                  <p>抓取入口就是淘宝的搜索页面，这个链接可以通过直接构造参数访问。例如，如果搜索iPad，就可以直接访问<a href="https://s.taobao.com/search?q=iPad" target="_blank" rel="noopener">https://s.taobao.com/search?q=iPad</a>，呈现的就是第一页的搜索结果，如图7-22所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-22.png" alt="">图7-22 搜索结果</p>
                  <p>在页面下方，有一个分页导航，其中既包括前5页的链接，也包括下一页的链接，同时还有一个输入任意页码跳转的链接，如图7-23所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-23-1.jpg" alt="">图7-23 分页导航</p>
                  <p>这里商品的搜索结果一般最大都为100页，要获取每一页的内容，只需要将页码从1到100顺序遍历即可，页码数是确定的。所以，直接在页面跳转文本框中输入要跳转的页码，然后点击“确定”按钮即可跳转到页码对应的页面。</p>
                  <p>这里不直接点击“下一页”的原因是：一旦爬取过程中出现异常退出，比如到50页退出了，此时点击“下一页”时，就无法快速切换到对应的后续页面了。此外，在爬取过程中，也需要记录当前的页码数，而且一旦点击“下一页”之后页面加载失败，还需要做异常检测，检测当前页面是加载到了第几页。整个流程相对比较复杂，所以这里我们直接用跳转的方式来爬取页面。</p>
                  <p>当我们成功加载出某一页商品列表时，利用Selenium即可获取页面源代码，然后再用相应的解析库解析即可。这里我们选用pyquery进行解析。下面我们用代码来实现整个抓取过程。</p>
                  <h2 id="5-获取商品列表"><a href="#5-获取商品列表" class="headerlink" title="5. 获取商品列表"></a>5. 获取商品列表</h2>
                  <p>首先，需要构造一个抓取的URL：<a href="https://s.taobao.com/search?q=iPad" target="_blank" rel="noopener">https://s.taobao.com/search?q=iPad</a>。这个URL非常简洁，参数<code>q</code>就是要搜索的关键字。只要改变这个参数，即可获取不同商品的列表。这里我们将商品的关键字定义成一个变量，然后构造出这样的一个URL。</p>
                  <p>然后，就需要用Selenium进行抓取了。我们实现如下抓取列表页的方法：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> selenium <span class="keyword">import</span> webdriver</span><br><span class="line"><span class="keyword">from</span> selenium.common.exceptions <span class="keyword">import</span> TimeoutException</span><br><span class="line"><span class="keyword">from</span> selenium.webdriver.common.by <span class="keyword">import</span> By</span><br><span class="line"><span class="keyword">from</span> selenium.webdriver.support <span class="keyword">import</span> expected_conditions <span class="keyword">as</span> EC</span><br><span class="line"><span class="keyword">from</span> selenium.webdriver.support.wait <span class="keyword">import</span> WebDriverWait</span><br><span class="line"><span class="keyword">from</span> urllib.parse <span class="keyword">import</span> quote</span><br><span class="line"></span><br><span class="line">browser = webdriver.Chrome()</span><br><span class="line">wait = WebDriverWait(browser, <span class="number">10</span>)</span><br><span class="line">KEYWORD = <span class="string">'iPad'</span></span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">index_page</span><span class="params">(page)</span>:</span></span><br><span class="line">    <span class="string">"""</span></span><br><span class="line"><span class="string">    抓取索引页</span></span><br><span class="line"><span class="string">    :param page: 页码</span></span><br><span class="line"><span class="string">    """</span></span><br><span class="line">    print(<span class="string">'正在爬取第'</span>, page, <span class="string">'页'</span>)</span><br><span class="line">    <span class="keyword">try</span>:</span><br><span class="line">        url = <span class="string">'https://s.taobao.com/search?q='</span> + quote(KEYWORD)</span><br><span class="line">        browser.get(url)</span><br><span class="line">        <span class="keyword">if</span> page &gt; <span class="number">1</span>:</span><br><span class="line">            input = wait.until(</span><br><span class="line">                EC.presence_of_element_located((By.CSS_SELECTOR, <span class="string">'#mainsrp-pager div.form &gt; input'</span>)))</span><br><span class="line">            submit = wait.until(</span><br><span class="line">                EC.element_to_be_clickable((By.CSS_SELECTOR, <span class="string">'#mainsrp-pager div.form &gt; span.btn.J_Submit'</span>)))</span><br><span class="line">            input.clear()</span><br><span class="line">            input.send_keys(page)</span><br><span class="line">            submit.click()</span><br><span class="line">        wait.until(</span><br><span class="line">            EC.text_to_be_present_in_element((By.CSS_SELECTOR, <span class="string">'#mainsrp-pager li.item.active &gt; span'</span>), str(page)))</span><br><span class="line">        wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, <span class="string">'.m-itemlist .items .item'</span>)))</span><br><span class="line">        get_products()</span><br><span class="line">    <span class="keyword">except</span> TimeoutException:</span><br><span class="line">        index_page(page)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里首先构造了一个<code>WebDriver</code>对象，使用的浏览器是Chrome，然后指定一个关键词，如iPad，接着定义了<code>index_page()</code>方法，用于抓取商品列表页。</p>
                  <p>在该方法里，我们首先访问了搜索商品的链接，然后判断了当前的页码，如果大于1，就进行跳页操作，否则等待页面加载完成。</p>
                  <p>等待加载时，我们使用了<code>WebDriverWait</code>对象，它可以指定等待条件，同时指定一个最长等待时间，这里指定为最长10秒。如果在这个时间内成功匹配了等待条件，也就是说页面元素成功加载出来了，就立即返回相应结果并继续向下执行，否则到了最大等待时间还没有加载出来时，就直接抛出超时异常。</p>
                  <p>比如，我们最终要等待商品信息加载出来，就指定了<code>presence_of_element_located</code>这个条件，然后传入了<code>.m-itemlist .items .item</code>这个选择器，而这个选择器对应的页面内容就是每个商品的信息块，可以到网页里面查看一下。如果加载成功，就会执行后续的<code>get_products()</code>方法，提取商品信息。</p>
                  <p>关于翻页操作，这里首先获取页码输入框，赋值为<code>input</code>，然后获取“确定”按钮，赋值为<code>submit</code>，分别是图7-24中的两个元素。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-24-1.jpg" alt="">图7-24 跳转选项</p>
                  <p>首先，我们清空了输入框，此时调用<code>clear()</code>方法即可。随后，调用<code>send_keys()</code>方法将页码填充到输入框中，然后点击“确定”按钮即可。</p>
                  <p>那么，怎样知道有没有跳转到对应的页码呢？我们可以注意到，成功跳转某一页后，页码都会高亮显示，如图7-25所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-25-1.jpg" alt="">图7-25 页码高亮显示</p>
                  <p>我们只需要判断当前高亮的页码数是当前的页码数即可，所以这里使用了另一个等待条件<code>text_to_be_present_in_element</code>，它会等待指定的文本出现在某一个节点里面时即返回成功。这里我们将高亮的页码节点对应的CSS选择器和当前要跳转的页码通过参数传递给这个等待条件，这样它就会检测当前高亮的页码节点是不是我们传过来的页码数，如果是，就证明页面成功跳转到了这一页，页面跳转成功。</p>
                  <p>这样刚才实现的<code>index_page()</code>方法就可以传入对应的页码，待加载出对应页码的商品列表后，再去调用<code>get_products()</code>方法进行页面解析。</p>
                  <h2 id="6-解析商品列表"><a href="#6-解析商品列表" class="headerlink" title="6. 解析商品列表"></a>6. 解析商品列表</h2>
                  <p>接下来，我们就可以实现<code>get_products()</code>方法来解析商品列表了。这里我们直接获取页面源代码，然后用pyquery进行解析，实现如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> pyquery import PyQuery as pq</span><br><span class="line">def get_products():</span><br><span class="line">    <span class="string">""</span><span class="string">"</span></span><br><span class="line"><span class="string">    提取商品数据</span></span><br><span class="line"><span class="string">    "</span><span class="string">""</span></span><br><span class="line">    html = browser.page_source</span><br><span class="line">    doc = pq(html)</span><br><span class="line">    items = doc(<span class="string">'#mainsrp-itemlist .items .item'</span>).items()</span><br><span class="line">    <span class="keyword">for</span> item <span class="keyword">in</span> items:</span><br><span class="line">        product = &#123;</span><br><span class="line">            <span class="string">'image'</span>: item.<span class="builtin-name">find</span>(<span class="string">'.pic .img'</span>).attr(<span class="string">'data-src'</span>),</span><br><span class="line">            <span class="string">'price'</span>: item.<span class="builtin-name">find</span>(<span class="string">'.price'</span>).text(),</span><br><span class="line">            <span class="string">'deal'</span>: item.<span class="builtin-name">find</span>(<span class="string">'.deal-cnt'</span>).text(),</span><br><span class="line">            <span class="string">'title'</span>: item.<span class="builtin-name">find</span>(<span class="string">'.title'</span>).text(),</span><br><span class="line">            <span class="string">'shop'</span>: item.<span class="builtin-name">find</span>(<span class="string">'.shop'</span>).text(),</span><br><span class="line">            <span class="string">'location'</span>: item.<span class="builtin-name">find</span>(<span class="string">'.location'</span>).text()</span><br><span class="line">        &#125;</span><br><span class="line">        <span class="builtin-name">print</span>(product)</span><br><span class="line">        save_to_mongo(product)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>首先，调用<code>page_source</code>属性获取页码的源代码，然后构造了PyQuery解析对象，接着提取了商品列表，此时使用的CSS选择器是<code>#mainsrp-itemlist .items .item</code>，它会匹配整个页面的每个商品。它的匹配结果是多个，所以这里我们又对它进行了一次遍历，用<code>for</code>循环将每个结果分别进行解析，每次循环把它赋值为<code>item</code>变量，每个<code>item</code>变量都是一个<code>PyQuery</code>对象，然后再调用它的<code>find()</code>方法，传入CSS选择器，就可以获取单个商品的特定内容了。</p>
                  <p>比如，查看一下商品信息的源码，如图7-26所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-26-1.jpg" alt="">图7-26 商品信息源码</p>
                  <p>可以发现，它是一个<code>img</code>节点，包含<code>id</code>、<code>class</code>、<code>data-src</code>、<code>alt</code>和<code>src</code>等属性。这里之所以可以看到这张图片，是因为它的<code>src</code>属性被赋值为图片的URL。把它的<code>src</code>属性提取出来，就可以获取商品的图片了。不过我们还注意<code>data-src</code>属性，它的内容也是图片的URL，观察后发现此URL是图片的完整大图，而<code>src</code>是压缩后的小图，所以这里抓取<code>data-src</code>属性来作为商品的图片。</p>
                  <p>因此，我们需要先利用<code>find()</code>方法找到图片的这个节点，然后再调用<code>attr()</code>方法获取商品的<code>data-src</code>属性，这样就成功提取了商品图片链接。然后用同样的方法提取商品的价格、成交量、名称、店铺和店铺所在地等信息，接着将所有提取结果赋值为一个字典<code>product</code>，随后调用<code>save_to_mongo()</code>将其保存到MongoDB即可。</p>
                  <h2 id="7-保存到MongoDB"><a href="#7-保存到MongoDB" class="headerlink" title="7. 保存到MongoDB"></a>7. 保存到MongoDB</h2>
                  <p>接下来，我们将商品信息保存到MongoDB，实现代码如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">MONGO_URL = <span class="string">'localhost'</span></span><br><span class="line">MONGO_DB = <span class="string">'taobao'</span></span><br><span class="line">MONGO_COLLECTION = <span class="string">'products'</span></span><br><span class="line">client = pymongo.MongoClient(MONGO_URL)</span><br><span class="line">db = client[MONGO_DB]</span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">save_to_mongo</span><span class="params">(result)</span>:</span></span><br><span class="line">    <span class="string">"""</span></span><br><span class="line"><span class="string">    保存至MongoDB</span></span><br><span class="line"><span class="string">    :param result: 结果</span></span><br><span class="line"><span class="string">    """</span></span><br><span class="line">    <span class="keyword">try</span>:</span><br><span class="line">        <span class="keyword">if</span> db[MONGO_COLLECTION].insert(result):</span><br><span class="line">            print(<span class="string">'存储到MongoDB成功'</span>)</span><br><span class="line">    <span class="keyword">except</span> Exception:</span><br><span class="line">        print(<span class="string">'存储到MongoDB失败'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里首先创建了一个MongoDB的连接对象，然后指定了数据库，随后指定了Collection的名称，接着直接调用<code>insert()</code>方法将数据插入到MongoDB。此处的<code>result</code>变量就是在<code>get_products()</code>方法里传来的<code>product</code>，包含单个商品的信息。</p>
                  <h2 id="8-遍历每页"><a href="#8-遍历每页" class="headerlink" title="8. 遍历每页"></a>8. 遍历每页</h2>
                  <p>刚才我们所定义的<code>get_index()</code>方法需要接收参数<code>page</code>，<code>page</code>代表页码。这里我们实现页码遍历即可，代码如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">MAX_PAGE = <span class="number">100</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">main</span><span class="params">()</span>:</span></span><br><span class="line">    <span class="string">"""</span></span><br><span class="line"><span class="string">    遍历每一页</span></span><br><span class="line"><span class="string">    """</span></span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> range(<span class="number">1</span>, MAX_PAGE + <span class="number">1</span>):</span><br><span class="line">        index_page(i)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>其实现非常简单，只需要调用一个<code>for</code>循环即可。这里定义最大的页码数为100，<code>range()</code>方法的返回结果就是1到100的列表，顺序遍历，调用<code>index_page()</code>方法即可。</p>
                  <p>这样我们的淘宝商品爬虫就完成了，最后调用<code>main()</code>方法即可运行。</p>
                  <h2 id="9-运行"><a href="#9-运行" class="headerlink" title="9. 运行"></a>9. 运行</h2>
                  <p>运行代码，可以发现首先会弹出一个Chrome浏览器，然后会访问淘宝页面，接着控制台便会输出相应的提取结果，如图7-27所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-27-1.jpg" alt="">图7-27 运行结果</p>
                  <p>可以发现，这些商品信息的结果都是字典形式，它们被存储到MongoDB里面。</p>
                  <p>再看一下MongoDB中的结果，如图7-28所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-28-1.jpg" alt="">图7-28 保存结果</p>
                  <p>可以看到，所有的信息都保存到MongoDB里了，这说明爬取成功。</p>
                  <h2 id="10-Chrome-Headless模式"><a href="#10-Chrome-Headless模式" class="headerlink" title="10. Chrome Headless模式"></a>10. Chrome Headless模式</h2>
                  <p>从Chrome 59版本开始，已经开始支持Headless模式，也就是无界面模式，这样爬取的时候就不会弹出浏览器了。如果要使用此模式，请把Chrome升级到59版本及以上。启用Headless模式的方式如下：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">chrome_options = webdriver.<span class="constructor">ChromeOptions()</span></span><br><span class="line">chrome_options.add<span class="constructor">_argument('--<span class="params">headless</span>')</span></span><br><span class="line">browser = webdriver.<span class="constructor">Chrome(<span class="params">chrome_options</span>=<span class="params">chrome_options</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>首先，创建<code>ChromeOptions</code>对象，接着添加<code>headless</code>参数，然后在初始化Chrome对象的时候通过<code>chrome_options</code>传递这个<code>ChromeOptions</code>对象，这样我们就可以成功启用Chrome的Headless模式了。</p>
                  <h2 id="11-对接Firefox"><a href="#11-对接Firefox" class="headerlink" title="11. 对接Firefox"></a>11. 对接Firefox</h2>
                  <p>要对接Firefox浏览器，非常简单，只需要更改一处即可：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">browser</span> = webdriver.Firefox()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里更改了<code>browser</code>对象的创建方式，这样爬取的时候就会使用Firefox浏览器了。</p>
                  <h2 id="12-对接PhantomJS"><a href="#12-对接PhantomJS" class="headerlink" title="12. 对接PhantomJS"></a>12. 对接PhantomJS</h2>
                  <p>如果不想使用Chrome的Headless模式，还可以使用PhantomJS（它是一个无界面浏览器）来抓取。抓取时，同样不会弹出窗口，还是只需要将<code>WebDriver</code>的声明修改一下即可：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">browser</span> = webdriver.PhantomJS()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外，它还支持命令行配置。比如，可以设置缓存和禁用图片加载的功能，进一步提高爬取效率：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">SERVICE_ARGS</span> = [<span class="string">'--load-images=false'</span>, <span class="string">'--disk-cache=true'</span>]</span><br><span class="line"><span class="attr">browser</span> = webdriver.PhantomJS(service_args=SERVICE_ARGS)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>最后，给出本节的代码地址：<a href="https://github.com/Python3WebSpider/TaobaoProduct" target="_blank" rel="noopener">https://github.com/Python3WebSpider/TaobaoProduct</a>。</p>
                  <p>本节中，我们用Selenium演示了淘宝页面的抓取。利用它，我们不用去分析Ajax请求，真正做到可见即可爬。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-31 22:10:36" itemprop="dateCreated datePublished" datetime="2018-01-31T22:10:36+08:00">2018-01-31</time>
                </span>
                <span id="/5657.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 7.4-使用Selenium爬取淘宝商品" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>6.6k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>6 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5654.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5654.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 7.3-Splash负载均衡配置</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>用Splash做页面抓取时，如果爬取的量非常大，任务非常多，用一个Splash服务来处理的话，未免压力太大了，此时可以考虑搭建一个负载均衡器来把压力分散到各个服务器上。这相当于多台机器多个服务共同参与任务的处理，可以减小单个Splash服务的压力。</p>
                  <h2 id="1-配置Splash服务"><a href="#1-配置Splash服务" class="headerlink" title="1. 配置Splash服务"></a>1. 配置Splash服务</h2>
                  <p>要搭建Splash负载均衡，首先要有多个Splash服务。假如这里在4台远程主机的8050端口上都开启了Splash服务，它们的服务地址分别为41.159.27.223:8050、41.159.27.221:8050、41.159.27.9:8050和41.159.117.119:8050，这4个服务完全一致，都是通过Docker的Splash镜像开启的。访问其中任何一个服务时，都可以使用Splash服务。</p>
                  <h2 id="2-配置负载均衡"><a href="#2-配置负载均衡" class="headerlink" title="2. 配置负载均衡"></a>2. 配置负载均衡</h2>
                  <p>接下来，可以选用任意一台带有公网IP的主机来配置负载均衡。首先，在这台主机上装好Nginx，然后修改Nginx的配置文件nginx.conf，添加如下内容：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">http &#123;</span><br><span class="line">    upstream splash &#123;</span><br><span class="line">        least_conn;</span><br><span class="line">        server <span class="number">41.159</span><span class="number">.27</span><span class="number">.223</span>:<span class="number">8050</span>;</span><br><span class="line">        server <span class="number">41.159</span><span class="number">.27</span><span class="number">.221</span>:<span class="number">8050</span>;</span><br><span class="line">        server <span class="number">41.159</span><span class="number">.27</span><span class="number">.9</span>:<span class="number">8050</span>;</span><br><span class="line">        server <span class="number">41.159</span><span class="number">.117</span><span class="number">.119</span>:<span class="number">8050</span>;</span><br><span class="line">    &#125;</span><br><span class="line">    server &#123;</span><br><span class="line">        listen <span class="number">8050</span>;</span><br><span class="line">        location / &#123;</span><br><span class="line">            proxy_pass http:<span class="comment">//splash;</span></span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们通过<code>upstream</code>字段定义了一个名字叫作<code>splash</code>的服务集群配置。其中<code>least_conn</code>代表最少链接负载均衡，它适合处理请求处理时间长短不一造成服务器过载的情况。</p>
                  <p>当然，我们也可以不指定配置，具体如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">upstream splash &#123;</span><br><span class="line">   <span class="built_in"> server </span>41.159.27.223:8050;</span><br><span class="line">   <span class="built_in"> server </span>41.159.27.221:8050;</span><br><span class="line">   <span class="built_in"> server </span>41.159.27.9:8050;</span><br><span class="line">   <span class="built_in"> server </span>41.159.117.119:8050;</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样默认以轮询策略实现负载均衡，每个服务器的压力相同。此策略适合服务器配置相当、无状态且短平快的服务使用。</p>
                  <p>另外，我们还可以指定权重，配置如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">upstream splash &#123;</span><br><span class="line">   <span class="built_in"> server </span>41.159.27.223:8050 <span class="attribute">weight</span>=4;</span><br><span class="line">   <span class="built_in"> server </span>41.159.27.221:8050 <span class="attribute">weight</span>=2;</span><br><span class="line">   <span class="built_in"> server </span>41.159.27.9:8050 <span class="attribute">weight</span>=2;</span><br><span class="line">   <span class="built_in"> server </span>41.159.117.119:8050 <span class="attribute">weight</span>=1;</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里<code>weight</code>参数指定各个服务的权重，权重越高，分配到处理的请求越多。假如不同的服务器配置差别比较大的话，可以使用此种配置。</p>
                  <p>最后，还有一种IP散列负载均衡，配置如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">upstream splash &#123;</span><br><span class="line">    ip_hash;</span><br><span class="line">   <span class="built_in"> server </span>41.159.27.223:8050;</span><br><span class="line">   <span class="built_in"> server </span>41.159.27.221:8050;</span><br><span class="line">   <span class="built_in"> server </span>41.159.27.9:8050;</span><br><span class="line">   <span class="built_in"> server </span>41.159.117.119:8050;</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>服务器根据请求客户端的IP地址进行散列计算，确保使用同一个服务器响应请求，这种策略适合有状态的服务，比如用户登录后访问某个页面的情形。对于Splash来说，不需要应用此设置。</p>
                  <p>我们可以根据不同的情形选用不同的配置，配置完成后重启一下Nginx服务：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">sudo nginx -s reload</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样直接访问Nginx所在服务器的8050端口，即可实现负载均衡了。</p>
                  <h2 id="3-配置认证"><a href="#3-配置认证" class="headerlink" title="3. 配置认证"></a>3. 配置认证</h2>
                  <p>现在Splash是可以公开访问的，如果不想让其公开访问，还可以配置认证，这仍然借助于Nginx。可以在<code>server</code>的<code>location</code>字段中添加<code>auth_basic</code>和<code>auth_basic_user_file</code>字段，具体配置如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">http &#123;</span><br><span class="line">    upstream splash &#123;</span><br><span class="line">        least_conn;</span><br><span class="line">        server <span class="number">41.159</span><span class="number">.27</span><span class="number">.223</span>:<span class="number">8050</span>;</span><br><span class="line">        server <span class="number">41.159</span><span class="number">.27</span><span class="number">.221</span>:<span class="number">8050</span>;</span><br><span class="line">        server <span class="number">41.159</span><span class="number">.27</span><span class="number">.9</span>:<span class="number">8050</span>;</span><br><span class="line">        server <span class="number">41.159</span><span class="number">.117</span><span class="number">.119</span>:<span class="number">8050</span>;</span><br><span class="line">    &#125;</span><br><span class="line">    server &#123;</span><br><span class="line">        listen <span class="number">8050</span>;</span><br><span class="line">        location / &#123;</span><br><span class="line">            proxy_pass http:<span class="comment">//splash;</span></span><br><span class="line">            auth_basic <span class="string">"Restricted"</span>;</span><br><span class="line">            auth_basic_user_file /etc/nginx/conf.d/.htpasswd;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里使用的用户名和密码配置放置在/etc/nginx/conf.d目录下，我们需要使用<code>htpasswd</code>命令创建。例如，创建一个用户名为<code>admin</code>的文件，相关命令如下：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">htpasswd</span> <span class="selector-tag">-c</span> <span class="selector-class">.htpasswd</span> <span class="selector-tag">admin</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来就会提示我们输入密码，输入两次之后，就会生成密码文件，其内容如下：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">cat</span> <span class="selector-class">.htpasswd</span> </span><br><span class="line"><span class="selector-tag">admin</span><span class="selector-pseudo">:5ZBxQr0rCqwbc</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>配置完成后，重启一下Nginx服务：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">sudo nginx -s reload</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样访问认证就成功配置好了。</p>
                  <h2 id="4-测试"><a href="#4-测试" class="headerlink" title="4. 测试"></a>4. 测试</h2>
                  <p>最后，我们可以用代码来测试一下负载均衡的配置，看看到底是不是每次请求会切换IP。利用<a href="http://httpbin.org/get" target="_blank" rel="noopener">http://httpbin.org/get</a>测试即可，实现代码如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line"><span class="keyword">from</span> urllib.parse import quote</span><br><span class="line">import re</span><br><span class="line"></span><br><span class="line">lua = <span class="string">''</span><span class="string">'</span></span><br><span class="line"><span class="string">function main(splash, args)</span></span><br><span class="line"><span class="string">  local treat = require("treat")</span></span><br><span class="line"><span class="string">  local response = splash:http_get("http://httpbin.org/get")</span></span><br><span class="line"><span class="string">  return treat.as_string(response.body)</span></span><br><span class="line"><span class="string">end</span></span><br><span class="line"><span class="string">'</span><span class="string">''</span></span><br><span class="line"></span><br><span class="line">url = <span class="string">'http://splash:8050/execute?lua_source='</span> + quote(lua)</span><br><span class="line">response = requests.<span class="builtin-name">get</span>(url, auth=(<span class="string">'admin'</span>, <span class="string">'admin'</span>))</span><br><span class="line">ip = re.search(<span class="string">'(\d+\.\d+\.\d+\.\d+)'</span>, response.text).group(1)</span><br><span class="line"><span class="builtin-name">print</span>(ip)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里URL中的splash字符串请自行替换成自己的Nginx服务器IP。这里我修改了Hosts，设置了splash为Nginx服务器IP。</p>
                  <p>多次运行代码之后，可以发现每次请求的IP都会变化，比如第一次的结果：</p>
                  <figure class="highlight accesslog">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">41.159.27.223</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>第二次的结果：</p>
                  <figure class="highlight accesslog">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">41.159.27.9</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这就说明负载均衡已经成功实现了。</p>
                  <p>本节中，我们成功实现了负载均衡的配置。配置负载均衡后，可以多个Splash服务共同合作，减轻单个服务的负载，这还是比较有用的。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-31 22:07:31" itemprop="dateCreated datePublished" datetime="2018-01-31T22:07:31+08:00">2018-01-31</time>
                </span>
                <span id="/5654.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 7.3-Splash负载均衡配置" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>2.9k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>3 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5638.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5638.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 7.2-Splash的使用</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>Splash是一个JavaScript渲染服务，是一个带有HTTP API的轻量级浏览器，同时它对接了Python中的Twisted和QT库。利用它，我们同样可以实现动态渲染页面的抓取。</p>
                  <h2 id="1-功能介绍"><a href="#1-功能介绍" class="headerlink" title="1. 功能介绍"></a>1. 功能介绍</h2>
                  <p>利用Splash，我们可以实现如下功能：</p>
                  <ul>
                    <li>异步方式处理多个网页渲染过程；</li>
                    <li>获取渲染后的页面的源代码或截图；</li>
                    <li>通过关闭图片渲染或者使用Adblock规则来加快页面渲染速度；</li>
                    <li>可执行特定的JavaScript脚本；</li>
                    <li>可通过Lua脚本来控制页面渲染过程；</li>
                    <li>获取渲染的详细过程并通过HAR（HTTP Archive）格式呈现。</li>
                  </ul>
                  <p>接下来，我们来了解一下它的具体用法。</p>
                  <h2 id="2-准备工作"><a href="#2-准备工作" class="headerlink" title="2. 准备工作"></a>2. 准备工作</h2>
                  <p>在开始之前，请确保已经正确安装好了Splash并可以正常运行服务。如果没有安装，可以参考第1章。</p>
                  <h2 id="3-实例引入"><a href="#3-实例引入" class="headerlink" title="3. 实例引入"></a>3. 实例引入</h2>
                  <p>首先，通过Splash提供的Web页面来测试其渲染过程。例如，我们在本机8050端口上运行了Splash服务，打开<a href="http://localhost:8050/" target="_blank" rel="noopener">http://localhost:8050/</a>即可看到其Web页面，如图7-6所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-6.png" alt="">图7-6 Web页面</p>
                  <p>在图7-6右侧，呈现的是一个渲染示例。可以看到，上方有一个输入框，默认是<a href="http://google.com/" target="_blank" rel="noopener">http://google.com</a>，这里换成百度测试一下，将内容更改为<a href="https://www.baidu.com/" target="_blank" rel="noopener">https://www.baidu.com</a>，然后点击Render me按钮开始渲染，结果如图7-7所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-7.png" alt="">图7-7 运行结果</p>
                  <p>可以看到，网页的返回结果呈现了渲染截图、HAR加载统计数据、网页的源代码。</p>
                  <p>通过HAR的结果可以看到，Splash执行了整个网页的渲染过程，包括CSS、JavaScript的加载等过程，呈现的页面和我们在浏览器中得到的结果完全一致。</p>
                  <p>那么，这个过程由什么来控制呢？重新返回首页，可以看到实际上是有一段脚本，内容如下：</p>
                  <figure class="highlight groovy">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">function main(splash, args)</span><br><span class="line">  <span class="keyword">assert</span>(<span class="string">splash:</span>go(args.url))</span><br><span class="line">  <span class="keyword">assert</span>(<span class="string">splash:</span>wait(<span class="number">0.5</span>))</span><br><span class="line">  <span class="keyword">return</span> &#123;</span><br><span class="line">    html = <span class="string">splash:</span>html(),</span><br><span class="line">    png = <span class="string">splash:</span>png(),</span><br><span class="line">    har = <span class="string">splash:</span>har(),</span><br><span class="line">  &#125;</span><br><span class="line">end</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这个脚本实际上是用Lua语言写的脚本。即使不懂这个语言的语法，但从脚本的表面意思，我们也可以大致了解到它首先调用<code>go()</code>方法去加载页面，然后调用<code>wait()</code>方法等待了一定时间，最后返回了页面的源码、截图和HAR信息。</p>
                  <p>到这里，我们大体了解了Splash是通过Lua脚本来控制了页面的加载过程的，加载过程完全模拟浏览器，最后可返回各种格式的结果，如网页源码和截图等。</p>
                  <p>接下来，我们就来了解Lua脚本的写法以及相关API的用法。</p>
                  <h2 id="4-Splash-Lua脚本"><a href="#4-Splash-Lua脚本" class="headerlink" title="4. Splash Lua脚本"></a>4. Splash Lua脚本</h2>
                  <p>Splash可以通过Lua脚本执行一系列渲染操作，这样我们就可以用Splash来模拟类似Chrome、PhantomJS的操作了。</p>
                  <p>首先，我们来了解一下Splash Lua脚本的入口和执行方式。</p>
                  <h3 id="入口及返回值"><a href="#入口及返回值" class="headerlink" title="入口及返回值"></a>入口及返回值</h3>
                  <p>首先，来看一个基本实例：</p>
                  <figure class="highlight maxima">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">function main(splash, <span class="built_in">args</span>)</span><br><span class="line">  splash:<span class="built_in">go</span>(<span class="string">"http://www.baidu.com"</span>)</span><br><span class="line">  splash:wait(<span class="number">0.5</span>)</span><br><span class="line">  <span class="built_in">local</span> <span class="built_in">title</span> = splash:evaljs(<span class="string">"document.title"</span>)</span><br><span class="line">  <span class="built_in">return</span> &#123;<span class="built_in">title</span>=<span class="built_in">title</span>&#125;</span><br><span class="line">end</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们将代码粘贴到刚才打开的<a href="http://localhost:8050/" target="_blank" rel="noopener">http://localhost:8050/</a>的代码编辑区域，然后点击Render me!按钮来测试一下。</p>
                  <p>我们看到它返回了网页的标题，如图7-8所示。这里我们通过<code>evaljs()</code>方法传入JavaScript脚本，而<code>document.title</code>的执行结果就是返回网页标题，执行完毕后将其赋值给一个<code>title</code>变量，随后将其返回。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-8.png" alt="">图7-8 运行结果</p>
                  <p>注意，我们在这里定义的方法名称叫作<code>main()</code>。这个名称必须是固定的，Splash会默认调用这个方法。</p>
                  <p>该方法的返回值既可以是字典形式，也可以是字符串形式，最后都会转化为Splash HTTP Response，例如：</p>
                  <figure class="highlight ada">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">function</span> <span class="title">main</span>(splash)</span><br><span class="line">    <span class="keyword">return</span> <span class="type">&#123;hello="world!"&#125;</span></span><br><span class="line"><span class="keyword">end</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>返回了一个字典形式的内容。例如：</p>
                  <figure class="highlight ada">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">function</span> <span class="title">main</span>(splash)</span><br><span class="line">    <span class="keyword">return</span> <span class="type">'hello'</span></span><br><span class="line"><span class="keyword">end</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>返回了一个字符串形式的内容。</p>
                  <h3 id="异步处理"><a href="#异步处理" class="headerlink" title="异步处理"></a>异步处理</h3>
                  <p>Splash支持异步处理，但是这里并没有显式指明回调方法，其回调的跳转是在Splash内部完成的。示例如下：</p>
                  <figure class="highlight lua">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span><span class="params">(splash, args)</span></span></span><br><span class="line">  <span class="keyword">local</span> example_urls = &#123;<span class="string">"www.baidu.com"</span>, <span class="string">"www.taobao.com"</span>, <span class="string">"www.zhihu.com"</span>&#125;</span><br><span class="line">  <span class="keyword">local</span> urls = args.urls <span class="keyword">or</span> example_urls</span><br><span class="line">  <span class="keyword">local</span> results = &#123;&#125;</span><br><span class="line">  <span class="keyword">for</span> index, url <span class="keyword">in</span> <span class="built_in">ipairs</span>(urls) <span class="keyword">do</span></span><br><span class="line">    <span class="keyword">local</span> ok, reason = splash:go(<span class="string">"http://"</span> .. url)</span><br><span class="line">    <span class="keyword">if</span> ok <span class="keyword">then</span></span><br><span class="line">      splash:wait(<span class="number">2</span>)</span><br><span class="line">      results[url] = splash:png()</span><br><span class="line">    <span class="keyword">end</span></span><br><span class="line">  <span class="keyword">end</span></span><br><span class="line">  <span class="keyword">return</span> results</span><br><span class="line"><span class="keyword">end</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果是3个站点的截图，如图7-9所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-9.png" alt="">图7-9 运行结果</p>
                  <p>在脚本内调用的<code>wait()</code>方法类似于Python中的<code>sleep()</code>，其参数为等待的秒数。当Splash执行到此方法时，它会转而去处理其他任务，然后在指定的时间过后再回来继续处理。</p>
                  <p>这里值得注意的是，Lua脚本中的字符串拼接和Python不同，它使用的是<code>..</code>操作符，而不是<code>+</code>。如果有必要，可以简单了解一下Lua脚本的语法，详见<a href="http://www.runoob.com/lua/lua-basic-syntax.html" target="_blank" rel="noopener">http://www.runoob.com/lua/lua-basic-syntax.html</a>。</p>
                  <p>另外，这里做了加载时的异常检测。<code>go()</code>方法会返回加载页面的结果状态，如果页面出现4xx或5xx状态码，<code>ok</code>变量就为空，就不会返回加载后的图片。</p>
                  <h2 id="5-Splash对象属性"><a href="#5-Splash对象属性" class="headerlink" title="5. Splash对象属性"></a>5. Splash对象属性</h2>
                  <p>我们注意到，前面例子中<code>main()</code>方法的第一个参数是<code>splash</code>，这个对象非常重要，它类似于Selenium中的<code>WebDriver</code>对象，我们可以调用它的一些属性和方法来控制加载过程。接下来，先看下它的属性。</p>
                  <h3 id="args"><a href="#args" class="headerlink" title="args"></a><code>args</code></h3>
                  <p>该属性可以获取加载时配置的参数，比如URL，如果为GET请求，它还可以获取GET请求参数；如果为POST请求，它可以获取表单提交的数据。Splash也支持使用第二个参数直接作为<code>args</code>，例如：</p>
                  <figure class="highlight lua">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span><span class="params">(splash, args)</span></span></span><br><span class="line">    <span class="keyword">local</span> url = args.url</span><br><span class="line"><span class="keyword">end</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里第二个参数<code>args</code>就相当于<code>splash.args</code>属性，以上代码等价于：</p>
                  <figure class="highlight lua">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span><span class="params">(splash)</span></span></span><br><span class="line">    <span class="keyword">local</span> url = splash.args.url</span><br><span class="line"><span class="keyword">end</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="js-enabled"><a href="#js-enabled" class="headerlink" title="js_enabled"></a><code>js_enabled</code></h3>
                  <p>这个属性是Splash的JavaScript执行开关，可以将其配置为<code>true</code>或<code>false</code>来控制是否执行JavaScript代码，默认为<code>true</code>。例如，这里禁止执行JavaScript代码：</p>
                  <figure class="highlight maxima">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">function main(splash, <span class="built_in">args</span>)</span><br><span class="line">  splash:<span class="built_in">go</span>(<span class="string">"https://www.baidu.com"</span>)</span><br><span class="line">  splash.js_enabled = <span class="literal">false</span></span><br><span class="line">  <span class="built_in">local</span> <span class="built_in">title</span> = splash:evaljs(<span class="string">"document.title"</span>)</span><br><span class="line">  <span class="built_in">return</span> &#123;<span class="built_in">title</span>=<span class="built_in">title</span>&#125;</span><br><span class="line">end</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接着我们重新调用了<code>evaljs()</code>方法执行JavaScript代码，此时运行结果就会抛出异常：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">    <span class="attr">"error"</span>: <span class="number">400</span>,</span><br><span class="line">    <span class="attr">"type"</span>: <span class="string">"ScriptError"</span>,</span><br><span class="line">    <span class="attr">"info"</span>: &#123;</span><br><span class="line">        <span class="attr">"type"</span>: <span class="string">"JS_ERROR"</span>,</span><br><span class="line">        <span class="attr">"js_error_message"</span>: <span class="literal">null</span>,</span><br><span class="line">        <span class="attr">"source"</span>: <span class="string">"[string \"function main(splash, args)\r...\"]"</span>,</span><br><span class="line">        <span class="attr">"message"</span>: <span class="string">"[string \"function main(splash, args)\r...\"]:4: unknown JS error: None"</span>,</span><br><span class="line">        <span class="attr">"line_number"</span>: <span class="number">4</span>,</span><br><span class="line">        <span class="attr">"error"</span>: <span class="string">"unknown JS error: None"</span>,</span><br><span class="line">        <span class="attr">"splash_method"</span>: <span class="string">"evaljs"</span></span><br><span class="line">    &#125;,</span><br><span class="line">    <span class="attr">"description"</span>: <span class="string">"Error happened while executing Lua script"</span></span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>不过一般来说，不用设置此属性，默认开启即可。</p>
                  <h3 id="resource-timeout"><a href="#resource-timeout" class="headerlink" title="resource_timeout"></a><code>resource_timeout</code></h3>
                  <p>此属性可以设置加载的超时时间，单位是秒。如果设置为0或<code>nil</code>（类似Python中的<code>None</code>），代表不检测超时。示例如下：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">function</span> <span class="function"><span class="title">main</span>(<span class="variable">splash</span>)</span></span><br><span class="line">    <span class="variable">splash.resource_timeout</span> = <span class="number">0.1</span></span><br><span class="line">    <span class="function"><span class="title"><span class="built_in">assert</span></span>(<span class="variable">splash</span>:<span class="title">go</span>(<span class="string">'https://www.taobao.com'</span>))</span></span><br><span class="line">    <span class="variable">return</span> <span class="variable">splash</span>:<span class="function"><span class="title">png</span>()</span></span><br><span class="line"><span class="variable">end</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>例如，这里将超时时间设置为0.1秒。如果在0.1秒之内没有得到响应，就会抛出异常，错误如下：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">    <span class="attr">"error"</span>: <span class="number">400</span>,</span><br><span class="line">    <span class="attr">"type"</span>: <span class="string">"ScriptError"</span>,</span><br><span class="line">    <span class="attr">"info"</span>: &#123;</span><br><span class="line">        <span class="attr">"error"</span>: <span class="string">"network5"</span>,</span><br><span class="line">        <span class="attr">"type"</span>: <span class="string">"LUA_ERROR"</span>,</span><br><span class="line">        <span class="attr">"line_number"</span>: <span class="number">3</span>,</span><br><span class="line">        <span class="attr">"source"</span>: <span class="string">"[string \"function main(splash)\r...\"]"</span>,</span><br><span class="line">        <span class="attr">"message"</span>: <span class="string">"Lua error: [string \"function main(splash)\r...\"]:3: network5"</span></span><br><span class="line">    &#125;,</span><br><span class="line">    <span class="attr">"description"</span>: <span class="string">"Error happened while executing Lua script"</span></span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>此属性适合在网页加载速度较慢的情况下设置。如果超过了某个时间无响应，则直接抛出异常并忽略即可。</p>
                  <h3 id="images-enabled"><a href="#images-enabled" class="headerlink" title="images_enabled"></a><code>images_enabled</code></h3>
                  <p>此属性可以设置图片是否加载，默认情况下是加载的。禁用该属性后，可以节省网络流量并提高网页加载速度。但是需要注意的是，禁用图片加载可能会影响JavaScript渲染。因为禁用图片之后，它的外层DOM节点的高度会受影响，进而影响DOM节点的位置。因此，如果JavaScript对图片节点有操作的话，其执行就会受到影响。</p>
                  <p>另外值得注意的是，Splash使用了缓存。如果一开始加载出来了网页图片，然后禁用了图片加载，再重新加载页面，之前加载好的图片可能还会显示出来，这时直接重启Splash即可。</p>
                  <p>禁用图片加载的示例如下：</p>
                  <figure class="highlight lua">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span><span class="params">(splash, args)</span></span></span><br><span class="line">  splash.images_enabled = <span class="literal">false</span></span><br><span class="line">  <span class="built_in">assert</span>(splash:go(<span class="string">'https://www.jd.com'</span>))</span><br><span class="line">  <span class="keyword">return</span> &#123;png=splash:png()&#125;</span><br><span class="line"><span class="keyword">end</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样返回的页面截图就不会带有任何图片，加载速度也会快很多。</p>
                  <h3 id="plugins-enabled"><a href="#plugins-enabled" class="headerlink" title="plugins_enabled"></a><code>plugins_enabled</code></h3>
                  <p>此属性可以控制浏览器插件（如Flash插件）是否开启。默认情况下，此属性是<code>false</code>，表示不开启。可以使用如下代码控制其开启和关闭：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">splash.plugins_enabled</span> = <span class="literal">true</span>/<span class="literal">false</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="scroll-position"><a href="#scroll-position" class="headerlink" title="scroll_position"></a><code>scroll_position</code></h3>
                  <p>通过设置此属性，我们可以控制页面上下或左右滚动。这是一个比较常用的属性，示例如下：</p>
                  <figure class="highlight lua">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span><span class="params">(splash, args)</span></span></span><br><span class="line">  <span class="built_in">assert</span>(splash:go(<span class="string">'https://www.taobao.com'</span>))</span><br><span class="line">  splash.scroll_position = &#123;y=<span class="number">400</span>&#125;</span><br><span class="line">  <span class="keyword">return</span> &#123;png=splash:png()&#125;</span><br><span class="line"><span class="keyword">end</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们就可以控制页面向下滚动400像素值，结果如图7-10所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-10.png" alt="">图7-10 运行结果</p>
                  <p>如果要让页面左右滚动，可以传入<code>x</code>参数，代码如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">splash.scroll_position = &#123;<span class="attribute">x</span>=100, <span class="attribute">y</span>=200&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="6-Splash对象的方法"><a href="#6-Splash对象的方法" class="headerlink" title="6. Splash对象的方法"></a>6. Splash对象的方法</h2>
                  <p>除了前面介绍的属性外，Splash对象还有如下方法。</p>
                  <h3 id="go"><a href="#go" class="headerlink" title="go()"></a><code>go()</code></h3>
                  <p>该方法用来请求某个链接，而且它可以模拟GET和POST请求，同时支持传入请求头、表单等数据，其用法如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">ok, reason = splash:go&#123;url, <span class="attribute">baseurl</span>=<span class="literal">nil</span>, <span class="attribute">headers</span>=<span class="literal">nil</span>, <span class="attribute">http_method</span>=<span class="string">"GET"</span>, <span class="attribute">body</span>=<span class="literal">nil</span>, <span class="attribute">formdata</span>=<span class="literal">nil</span>&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>其参数说明如下。</p>
                  <ul>
                    <li><strong><code>url</code></strong>：请求的URL。</li>
                    <li><strong><code>baseurl</code></strong>：可选参数，默认为空，表示资源加载相对路径。</li>
                    <li><strong><code>headers</code></strong>：可选参数，默认为空，表示请求头。</li>
                    <li><strong><code>http_method</code></strong>：可选参数，默认为<code>GET</code>，同时支持<code>POST</code>。</li>
                    <li><strong><code>body</code></strong>：可选参数，默认为空，发POST请求时的表单数据，使用的<code>Content-type</code>为<code>application/json</code>。</li>
                    <li><strong><code>formdata</code></strong>：可选参数，默认为空，POST的时候的表单数据，使用的<code>Content-type</code>为<code>application/x-www-form-urlencoded</code>。</li>
                  </ul>
                  <p>该方法的返回结果是结果<code>ok</code>和原因<code>reason</code>的组合，如果<code>ok</code>为空，代表网页加载出现了错误，此时<code>reason</code>变量中包含了错误的原因，否则证明页面加载成功。示例如下：</p>
                  <figure class="highlight lua">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span><span class="params">(splash, args)</span></span></span><br><span class="line">  <span class="keyword">local</span> ok, reason = splash:go&#123;<span class="string">"http://httpbin.org/post"</span>, http_method=<span class="string">"POST"</span>, body=<span class="string">"name=Germey"</span>&#125;</span><br><span class="line">  <span class="keyword">if</span> ok <span class="keyword">then</span></span><br><span class="line">        <span class="keyword">return</span> splash:html()</span><br><span class="line">  <span class="keyword">end</span></span><br><span class="line"><span class="keyword">end</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们模拟了一个POST请求，并传入了POST的表单数据，如果成功，则返回页面的源代码。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight dts">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="params">&lt;html&gt;</span><span class="params">&lt;head&gt;</span><span class="params">&lt;/head&gt;</span><span class="params">&lt;body&gt;</span><span class="params">&lt;pre style="word-wrap: break-word; white-space: pre-wrap;"&gt;</span>&#123;</span><br><span class="line">  <span class="string">"args"</span>: &#123;&#125;, </span><br><span class="line">  <span class="string">"data"</span>: <span class="string">""</span>, </span><br><span class="line">  <span class="string">"files"</span>: &#123;&#125;, </span><br><span class="line">  <span class="string">"form"</span>: &#123;</span><br><span class="line">    <span class="string">"name"</span>: <span class="string">"Germey"</span></span><br><span class="line">  &#125;, </span><br><span class="line">  <span class="string">"headers"</span>: &#123;</span><br><span class="line">    <span class="string">"Accept"</span>: <span class="string">"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"</span>, </span><br><span class="line">    <span class="string">"Accept-Encoding"</span>: <span class="string">"gzip, deflate"</span>, </span><br><span class="line">    <span class="string">"Accept-Language"</span>: <span class="string">"en,*"</span>, </span><br><span class="line">    <span class="string">"Connection"</span>: <span class="string">"close"</span>, </span><br><span class="line">    <span class="string">"Content-Length"</span>: <span class="string">"11"</span>, </span><br><span class="line">    <span class="string">"Content-Type"</span>: <span class="string">"application/x-www-form-urlencoded"</span>, </span><br><span class="line">    <span class="string">"Host"</span>: <span class="string">"httpbin.org"</span>, </span><br><span class="line">    <span class="string">"Origin"</span>: <span class="string">"null"</span>, </span><br><span class="line">    <span class="string">"User-Agent"</span>: <span class="string">"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/602.1 (KHTML, like Gecko) splash Version/9.0 Safari/602.1"</span></span><br><span class="line">  &#125;, </span><br><span class="line">  <span class="string">"json"</span>: null, </span><br><span class="line">  <span class="string">"origin"</span>: <span class="string">"60.207.237.85"</span>, </span><br><span class="line">  <span class="string">"url"</span>: <span class="string">"http://httpbin.org/post"</span></span><br><span class="line">&#125;</span><br><span class="line"><span class="params">&lt;/pre&gt;</span><span class="params">&lt;/body&gt;</span><span class="params">&lt;/html&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，我们成功实现了POST请求并发送了表单数据。</p>
                  <h3 id="wait"><a href="#wait" class="headerlink" title="wait()"></a><code>wait()</code></h3>
                  <p>此方法可以控制页面的等待时间，使用方法如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">ok, reason = splash:wait&#123;time, <span class="attribute">cancel_on_redirect</span>=<span class="literal">false</span>, <span class="attribute">cancel_on_error</span>=<span class="literal">true</span>&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>参数说明如下。</p>
                  <ul>
                    <li><strong><code>time</code></strong>：等待的秒数。</li>
                    <li><strong><code>cancel_on_redirect</code></strong>：可选参数，默认为<code>false</code>，表示如果发生了重定向就停止等待，并返回重定向结果。</li>
                    <li><strong><code>cancel_on_error</code></strong>：可选参数，默认为<code>false</code>，表示如果发生了加载错误，就停止等待。</li>
                  </ul>
                  <p>返回结果同样是结果<code>ok</code>和原因<code>reason</code>的组合。</p>
                  <p>我们用一个实例感受一下：</p>
                  <figure class="highlight actionscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span><span class="params">(splash)</span></span></span><br><span class="line"><span class="function">    <span class="title">splash</span>:go<span class="params">(<span class="string">"https://www.taobao.com"</span>)</span></span></span><br><span class="line"><span class="function">    <span class="title">splash</span>:wait<span class="params">(2)</span></span></span><br><span class="line"><span class="function">    <span class="title">return</span> </span>&#123;html=splash:html()&#125;</span><br><span class="line">end</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这可以实现访问淘宝并等待2秒，随后返回页面源代码的功能。</p>
                  <h3 id="jsfunc"><a href="#jsfunc" class="headerlink" title="jsfunc()"></a><code>jsfunc()</code></h3>
                  <p>此方法可以直接调用JavaScript定义的方法，但是所调用的方法需要用双中括号包围，这相当于实现了JavaScript方法到Lua脚本的转换。示例如下：</p>
                  <figure class="highlight typescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span>(<span class="params">splash, args</span>)</span></span><br><span class="line"><span class="function">  <span class="title">local</span> <span class="title">get_div_count</span> = <span class="title">splash</span>:<span class="title">jsfunc</span>(<span class="params">[[</span></span></span><br><span class="line"><span class="function"><span class="params">  <span class="keyword">function</span> () &#123;</span></span></span><br><span class="line"><span class="function"><span class="params">    <span class="keyword">var</span> body = <span class="built_in">document</span>.body;</span></span></span><br><span class="line"><span class="function"><span class="params">    <span class="keyword">var</span> divs = body.getElementsByTagName(<span class="string">'div'</span>);</span></span></span><br><span class="line"><span class="function"><span class="params">    <span class="keyword">return</span> divs.length;</span></span></span><br><span class="line"><span class="function"><span class="params">  &#125;</span></span></span><br><span class="line"><span class="function"><span class="params">  ]]</span>)</span></span><br><span class="line"><span class="function">  <span class="title">splash</span>:<span class="title">go</span>(<span class="params">"https:<span class="comment">//www.baidu.com")</span></span></span></span><br><span class="line"><span class="function"><span class="params">  <span class="keyword">return</span> (<span class="string">"There are %s DIVs"</span>):format(</span></span></span><br><span class="line"><span class="function"><span class="params">    get_div_count())</span></span></span><br><span class="line"><span class="function"><span class="params">end</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">There are <span class="number">21</span> DIVs</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>首先，我们声明了一个JavaScript定义的方法，然后在页面加载成功后调用了此方法计算出了页面中<code>div</code>节点的个数。</p>
                  <p>关于JavaScript到Lua脚本的更多转换细节，可以参考官方文档：<a href="https://splash.readthedocs.io/en/stable/scripting-ref.html#splash-jsfunc" target="_blank" rel="noopener">https://splash.readthedocs.io/en/stable/scripting-ref.html#splash-jsfunc</a>。</p>
                  <h3 id="evaljs"><a href="#evaljs" class="headerlink" title="evaljs()"></a><code>evaljs()</code></h3>
                  <p>此方法可以执行JavaScript代码并返回最后一条JavaScript语句的返回结果，使用方法如下：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable"><span class="class">result</span></span> = <span class="variable">splash</span>:<span class="function"><span class="title">evaljs</span>(<span class="variable">js</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>比如，可以用下面的代码来获取页面标题：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">local</span> <span class="variable">title</span> = <span class="variable">splash</span>:<span class="function"><span class="title">evaljs</span>(<span class="string">"document.title"</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="runjs"><a href="#runjs" class="headerlink" title="runjs()"></a><code>runjs()</code></h3>
                  <p>此方法可以执行JavaScript代码，它与<code>evaljs()</code>的功能类似，但是更偏向于执行某些动作或声明某些方法。例如：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span>(<span class="title">splash</span>, <span class="title">args</span>)</span></span><br><span class="line">  splash:go(<span class="string">"https://www.baidu.com"</span>)</span><br><span class="line">  splash:runjs(<span class="string">"foo = function() &#123; return 'bar' &#125;"</span>)</span><br><span class="line">  <span class="built_in">local</span> <span class="built_in">result</span> = splash:evaljs(<span class="string">"foo()"</span>)</span><br><span class="line">  <span class="literal">return</span> <span class="built_in">result</span></span><br><span class="line"><span class="keyword">end</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们用<code>runjs()</code>先声明了一个JavaScript定义的方法，然后通过<code>evaljs()</code>来调用得到的结果。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">bar</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="autoload"><a href="#autoload" class="headerlink" title="autoload()"></a><code>autoload()</code></h3>
                  <p>此方法可以设置每个页面访问时自动加载的对象，使用方法如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">ok, reason = splash:autoload&#123;source_or_url, <span class="attribute">source</span>=<span class="literal">nil</span>, <span class="attribute">url</span>=<span class="literal">nil</span>&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>参数说明如下。</p>
                  <ul>
                    <li><strong><code>source_or_url</code></strong>：JavaScript代码或者JavaScript库链接。</li>
                    <li><strong><code>source</code></strong>：JavaScript代码。</li>
                    <li><strong><code>url</code></strong>：JavaScript库链接</li>
                  </ul>
                  <p>但是此方法只负责加载JavaScript代码或库，不执行任何操作。如果要执行操作，可以调用<code>evaljs()</code>或<code>runjs()</code>方法。示例如下：</p>
                  <figure class="highlight lua">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span><span class="params">(splash, args)</span></span></span><br><span class="line">  splash:autoload(<span class="string">[[</span></span><br><span class="line"><span class="string">    function get_document_title()&#123;</span></span><br><span class="line"><span class="string">      return document.title;</span></span><br><span class="line"><span class="string">    &#125;</span></span><br><span class="line"><span class="string">  ]]</span>)</span><br><span class="line">  splash:go(<span class="string">"https://www.baidu.com"</span>)</span><br><span class="line">  <span class="keyword">return</span> splash:evaljs(<span class="string">"get_document_title()"</span>)</span><br><span class="line"><span class="keyword">end</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们调用<code>autoload()</code>方法声明了一个JavaScript方法，然后通过<code>evaljs()</code>方法来执行此JavaScript方法。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight plain">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">百度一下，你就知道</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外，我们也可以使用<code>autoload()</code>方法加载某些方法库，如jQuery，示例如下：</p>
                  <figure class="highlight lua">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span><span class="params">(splash, args)</span></span></span><br><span class="line">  <span class="built_in">assert</span>(splash:autoload(<span class="string">"https://code.jquery.com/jquery-2.1.3.min.js"</span>))</span><br><span class="line">  <span class="built_in">assert</span>(splash:go(<span class="string">"https://www.taobao.com"</span>))</span><br><span class="line">  <span class="keyword">local</span> version = splash:evaljs(<span class="string">"$.fn.jquery"</span>)</span><br><span class="line">  <span class="keyword">return</span> <span class="string">'JQuery version: '</span> .. version</span><br><span class="line"><span class="keyword">end</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight yaml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">JQuery version:</span> <span class="number">2.1</span><span class="number">.3</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="call-later"><a href="#call-later" class="headerlink" title="call_later()"></a><code>call_later()</code></h3>
                  <p>此方法可以通过设置定时任务和延迟时间来实现任务延时执行，并且可以在执行前通过<code>cancel()</code>方法重新执行定时任务。示例如下：</p>
                  <figure class="highlight lua">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span><span class="params">(splash, args)</span></span></span><br><span class="line">  <span class="keyword">local</span> snapshots = &#123;&#125;</span><br><span class="line">  <span class="keyword">local</span> timer = splash:call_later(<span class="function"><span class="keyword">function</span><span class="params">()</span></span></span><br><span class="line">    snapshots[<span class="string">"a"</span>] = splash:png()</span><br><span class="line">    splash:wait(<span class="number">1.0</span>)</span><br><span class="line">    snapshots[<span class="string">"b"</span>] = splash:png()</span><br><span class="line">  <span class="keyword">end</span>, <span class="number">0.2</span>)</span><br><span class="line">  splash:go(<span class="string">"https://www.taobao.com"</span>)</span><br><span class="line">  splash:wait(<span class="number">3.0</span>)</span><br><span class="line">  <span class="keyword">return</span> snapshots</span><br><span class="line"><span class="keyword">end</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们设置了一个定时任务，0.2秒的时候获取网页截图，然后等待1秒，1.2秒时再次获取网页截图，访问的页面是淘宝，最后将截图结果返回。运行结果如图7-11所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-11.png" alt="">图7-11 运行结果</p>
                  <p>可以发现，第一次截图时网页还没有加载出来，截图为空，第二次网页便加载成功了。</p>
                  <h3 id="http-get"><a href="#http-get" class="headerlink" title="http_get()"></a><code>http_get()</code></h3>
                  <p>此方法可以模拟发送HTTP的GET请求，使用方法如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">response = splash:http_get&#123;url, <span class="attribute">headers</span>=<span class="literal">nil</span>, <span class="attribute">follow_redirects</span>=<span class="literal">true</span>&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>参数说明如下。</p>
                  <ul>
                    <li><strong><code>url</code></strong>：请求URL。</li>
                    <li><strong><code>headers</code></strong>：可选参数，默认为空，请求头。</li>
                    <li><strong><code>follow_redirects</code></strong>：可选参数，表示是否启动自动重定向，默认为<code>true</code>。</li>
                  </ul>
                  <p>示例如下：</p>
                  <figure class="highlight lua">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span><span class="params">(splash, args)</span></span></span><br><span class="line">  <span class="keyword">local</span> treat = <span class="built_in">require</span>(<span class="string">"treat"</span>)</span><br><span class="line">  <span class="keyword">local</span> response = splash:http_get(<span class="string">"http://httpbin.org/get"</span>)</span><br><span class="line">    <span class="keyword">return</span> &#123;</span><br><span class="line">    html=treat.as_string(response.body),</span><br><span class="line">    url=response.url,</span><br><span class="line">    <span class="built_in">status</span>=response.<span class="built_in">status</span></span><br><span class="line">    &#125;</span><br><span class="line"><span class="keyword">end</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight groovy">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Splash <span class="string">Response:</span> Object</span><br><span class="line"><span class="string">html:</span> String (length <span class="number">355</span>)</span><br><span class="line">&#123;</span><br><span class="line">  <span class="string">"args"</span>: &#123;&#125;, </span><br><span class="line">  <span class="string">"headers"</span>: &#123;</span><br><span class="line">    <span class="string">"Accept-Encoding"</span>: <span class="string">"gzip, deflate"</span>, </span><br><span class="line">    <span class="string">"Accept-Language"</span>: <span class="string">"en,*"</span>, </span><br><span class="line">    <span class="string">"Connection"</span>: <span class="string">"close"</span>, </span><br><span class="line">    <span class="string">"Host"</span>: <span class="string">"httpbin.org"</span>, </span><br><span class="line">    <span class="string">"User-Agent"</span>: <span class="string">"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/602.1 (KHTML, like Gecko) splash Version/9.0 Safari/602.1"</span></span><br><span class="line">  &#125;, </span><br><span class="line">  <span class="string">"origin"</span>: <span class="string">"60.207.237.85"</span>, </span><br><span class="line">  <span class="string">"url"</span>: <span class="string">"http://httpbin.org/get"</span></span><br><span class="line">&#125;</span><br><span class="line"><span class="string">status:</span> <span class="number">200</span></span><br><span class="line"><span class="string">url:</span> <span class="string">"http://httpbin.org/get"</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="http-post"><a href="#http-post" class="headerlink" title="http_post()"></a><code>http_post()</code></h3>
                  <p>和<code>http_get()</code>方法类似，此方法用来模拟发送POST请求，不过多了一个参数<code>body</code>，使用方法如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">response = splash:http_post&#123;url, <span class="attribute">headers</span>=<span class="literal">nil</span>, <span class="attribute">follow_redirects</span>=<span class="literal">true</span>, <span class="attribute">body</span>=<span class="literal">nil</span>&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>参数说明如下。</p>
                  <ul>
                    <li><strong><code>url</code></strong>：请求URL。</li>
                    <li><strong><code>headers</code></strong>：可选参数，默认为空，请求头。</li>
                    <li><strong><code>follow_redirects</code></strong>：可选参数，表示是否启动自动重定向，默认为<code>true</code>。</li>
                    <li><strong><code>body</code></strong>：可选参数，即表单数据，默认为空。</li>
                  </ul>
                  <p>我们用实例感受一下：</p>
                  <figure class="highlight lua">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span><span class="params">(splash, args)</span></span></span><br><span class="line">  <span class="keyword">local</span> treat = <span class="built_in">require</span>(<span class="string">"treat"</span>)</span><br><span class="line">  <span class="keyword">local</span> json = <span class="built_in">require</span>(<span class="string">"json"</span>)</span><br><span class="line">  <span class="keyword">local</span> response = splash:http_post&#123;<span class="string">"http://httpbin.org/post"</span>,     </span><br><span class="line">      body=json.encode(&#123;name=<span class="string">"Germey"</span>&#125;),</span><br><span class="line">      headers=&#123;[<span class="string">"content-type"</span>]=<span class="string">"application/json"</span>&#125;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">return</span> &#123;</span><br><span class="line">    html=treat.as_string(response.body),</span><br><span class="line">    url=response.url,</span><br><span class="line">    <span class="built_in">status</span>=response.<span class="built_in">status</span></span><br><span class="line">    &#125;</span><br><span class="line"><span class="keyword">end</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight groovy">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Splash <span class="string">Response:</span> Object</span><br><span class="line"><span class="string">html:</span> String (length <span class="number">533</span>)</span><br><span class="line">&#123;</span><br><span class="line">  <span class="string">"args"</span>: &#123;&#125;, </span><br><span class="line">  <span class="string">"data"</span>: <span class="string">"&#123;\"name\": \"Germey\"&#125;"</span>, </span><br><span class="line">  <span class="string">"files"</span>: &#123;&#125;, </span><br><span class="line">  <span class="string">"form"</span>: &#123;&#125;, </span><br><span class="line">  <span class="string">"headers"</span>: &#123;</span><br><span class="line">    <span class="string">"Accept-Encoding"</span>: <span class="string">"gzip, deflate"</span>, </span><br><span class="line">    <span class="string">"Accept-Language"</span>: <span class="string">"en,*"</span>, </span><br><span class="line">    <span class="string">"Connection"</span>: <span class="string">"close"</span>, </span><br><span class="line">    <span class="string">"Content-Length"</span>: <span class="string">"18"</span>, </span><br><span class="line">    <span class="string">"Content-Type"</span>: <span class="string">"application/json"</span>, </span><br><span class="line">    <span class="string">"Host"</span>: <span class="string">"httpbin.org"</span>, </span><br><span class="line">    <span class="string">"User-Agent"</span>: <span class="string">"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/602.1 (KHTML, like Gecko) splash Version/9.0 Safari/602.1"</span></span><br><span class="line">  &#125;, </span><br><span class="line">  <span class="string">"json"</span>: &#123;</span><br><span class="line">    <span class="string">"name"</span>: <span class="string">"Germey"</span></span><br><span class="line">  &#125;, </span><br><span class="line">  <span class="string">"origin"</span>: <span class="string">"60.207.237.85"</span>, </span><br><span class="line">  <span class="string">"url"</span>: <span class="string">"http://httpbin.org/post"</span></span><br><span class="line">&#125;</span><br><span class="line"><span class="string">status:</span> <span class="number">200</span></span><br><span class="line"><span class="string">url:</span> <span class="string">"http://httpbin.org/post"</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，这里我们成功模拟提交了POST请求并发送了表单数据。</p>
                  <h3 id="set-content"><a href="#set-content" class="headerlink" title="set_content()"></a><code>set_content()</code></h3>
                  <p>此方法用来设置页面的内容，示例如下：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">function</span> <span class="function"><span class="title">main</span>(<span class="variable">splash</span>)</span></span><br><span class="line">    <span class="function"><span class="title"><span class="built_in">assert</span></span>(<span class="variable">splash</span>:<span class="title">set_content</span>(<span class="string">"&lt;html&gt;&lt;body&gt;&lt;h1&gt;hello&lt;/h1&gt;&lt;/body&gt;&lt;/html&gt;"</span>))</span></span><br><span class="line">    <span class="variable">return</span> <span class="variable">splash</span>:<span class="function"><span class="title">png</span>()</span></span><br><span class="line"><span class="variable">end</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如图7-12所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-12.png" alt="">图7-12 运行结果</p>
                  <h3 id="html"><a href="#html" class="headerlink" title="html()"></a><code>html()</code></h3>
                  <p>此方法用来获取网页的源代码，它是非常简单又常用的方法。示例如下：</p>
                  <figure class="highlight actionscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span><span class="params">(splash, args)</span></span></span><br><span class="line"><span class="function">  <span class="title">splash</span>:go<span class="params">(<span class="string">"https://httpbin.org/get"</span>)</span></span></span><br><span class="line"><span class="function">  <span class="title">return</span> <span class="title">splash</span>:html<span class="params">()</span></span></span><br><span class="line"><span class="function"><span class="title">end</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight dts">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="params">&lt;html&gt;</span><span class="params">&lt;head&gt;</span><span class="params">&lt;/head&gt;</span><span class="params">&lt;body&gt;</span><span class="params">&lt;pre style="word-wrap: break-word; white-space: pre-wrap;"&gt;</span>&#123;</span><br><span class="line">  <span class="string">"args"</span>: &#123;&#125;, </span><br><span class="line">  <span class="string">"headers"</span>: &#123;</span><br><span class="line">    <span class="string">"Accept"</span>: <span class="string">"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"</span>, </span><br><span class="line">    <span class="string">"Accept-Encoding"</span>: <span class="string">"gzip, deflate"</span>, </span><br><span class="line">    <span class="string">"Accept-Language"</span>: <span class="string">"en,*"</span>, </span><br><span class="line">    <span class="string">"Connection"</span>: <span class="string">"close"</span>, </span><br><span class="line">    <span class="string">"Host"</span>: <span class="string">"httpbin.org"</span>, </span><br><span class="line">    <span class="string">"User-Agent"</span>: <span class="string">"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/602.1 (KHTML, like Gecko) splash Version/9.0 Safari/602.1"</span></span><br><span class="line">  &#125;, </span><br><span class="line">  <span class="string">"origin"</span>: <span class="string">"60.207.237.85"</span>, </span><br><span class="line">  <span class="string">"url"</span>: <span class="string">"https://httpbin.org/get"</span></span><br><span class="line">&#125;</span><br><span class="line"><span class="params">&lt;/pre&gt;</span><span class="params">&lt;/body&gt;</span><span class="params">&lt;/html&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="png"><a href="#png" class="headerlink" title="png()"></a><code>png()</code></h3>
                  <p>此方法用来获取PNG格式的网页截图，示例如下：</p>
                  <figure class="highlight actionscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span><span class="params">(splash, args)</span></span></span><br><span class="line"><span class="function">  <span class="title">splash</span>:go<span class="params">(<span class="string">"https://www.taobao.com"</span>)</span></span></span><br><span class="line"><span class="function">  <span class="title">return</span> <span class="title">splash</span>:png<span class="params">()</span></span></span><br><span class="line"><span class="function"><span class="title">end</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="jpeg"><a href="#jpeg" class="headerlink" title="jpeg()"></a><code>jpeg()</code></h3>
                  <p>此方法用来获取JPEG格式的网页截图，示例如下：</p>
                  <figure class="highlight actionscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span><span class="params">(splash, args)</span></span></span><br><span class="line"><span class="function">  <span class="title">splash</span>:go<span class="params">(<span class="string">"https://www.taobao.com"</span>)</span></span></span><br><span class="line"><span class="function">  <span class="title">return</span> <span class="title">splash</span>:jpeg<span class="params">()</span></span></span><br><span class="line"><span class="function"><span class="title">end</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="har"><a href="#har" class="headerlink" title="har()"></a><code>har()</code></h3>
                  <p>此方法用来获取页面加载过程描述，示例如下：</p>
                  <figure class="highlight actionscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span><span class="params">(splash, args)</span></span></span><br><span class="line"><span class="function">  <span class="title">splash</span>:go<span class="params">(<span class="string">"https://www.baidu.com"</span>)</span></span></span><br><span class="line"><span class="function">  <span class="title">return</span> <span class="title">splash</span>:har<span class="params">()</span></span></span><br><span class="line"><span class="function"><span class="title">end</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如图7-13所示，其中显示了页面加载过程中每个请求记录的详情。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-13.png" alt="">图7-13 运行结果</p>
                  <h3 id="url"><a href="#url" class="headerlink" title="url()"></a><code>url()</code></h3>
                  <p>此方法可以获取当前正在访问的URL，示例如下：</p>
                  <figure class="highlight actionscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span><span class="params">(splash, args)</span></span></span><br><span class="line"><span class="function">  <span class="title">splash</span>:go<span class="params">(<span class="string">"https://www.baidu.com"</span>)</span></span></span><br><span class="line"><span class="function">  <span class="title">return</span> <span class="title">splash</span>:url<span class="params">()</span></span></span><br><span class="line"><span class="function"><span class="title">end</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight dts">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">https:</span><span class="comment">//www.baidu.com/</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="get-cookies"><a href="#get-cookies" class="headerlink" title="get_cookies()"></a><code>get_cookies()</code></h3>
                  <p>此方法可以获取当前页面的Cookies，示例如下：</p>
                  <figure class="highlight actionscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span><span class="params">(splash, args)</span></span></span><br><span class="line"><span class="function">  <span class="title">splash</span>:go<span class="params">(<span class="string">"https://www.baidu.com"</span>)</span></span></span><br><span class="line"><span class="function">  <span class="title">return</span> <span class="title">splash</span>:get_cookies<span class="params">()</span></span></span><br><span class="line"><span class="function"><span class="title">end</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight groovy">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Splash <span class="string">Response:</span> Array[<span class="number">2</span>]</span><br><span class="line"><span class="number">0</span>: Object</span><br><span class="line"><span class="string">domain:</span> <span class="string">".baidu.com"</span></span><br><span class="line"><span class="string">expires:</span> <span class="string">"2085-08-21T20:13:23Z"</span></span><br><span class="line"><span class="string">httpOnly:</span> <span class="literal">false</span></span><br><span class="line"><span class="string">name:</span> <span class="string">"BAIDUID"</span></span><br><span class="line"><span class="string">path:</span> <span class="string">"/"</span></span><br><span class="line"><span class="string">secure:</span> <span class="literal">false</span></span><br><span class="line"><span class="string">value:</span> <span class="string">"C1263A470B02DEF45593B062451C9722:FG=1"</span></span><br><span class="line"><span class="number">1</span>: Object</span><br><span class="line"><span class="string">domain:</span> <span class="string">".baidu.com"</span></span><br><span class="line"><span class="string">expires:</span> <span class="string">"2085-08-21T20:13:23Z"</span></span><br><span class="line"><span class="string">httpOnly:</span> <span class="literal">false</span></span><br><span class="line"><span class="string">name:</span> <span class="string">"BIDUPSID"</span></span><br><span class="line"><span class="string">path:</span> <span class="string">"/"</span></span><br><span class="line"><span class="string">secure:</span> <span class="literal">false</span></span><br><span class="line"><span class="string">value:</span> <span class="string">"C1263A470B02DEF45593B062451C9722"</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="add-cookie"><a href="#add-cookie" class="headerlink" title="add_cookie()"></a><code>add_cookie()</code></h3>
                  <p>此方法可以为当前页面添加Cookie，用法如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">cookies = splash:add_cookie&#123;name, value, <span class="attribute">path</span>=<span class="literal">nil</span>, <span class="attribute">domain</span>=<span class="literal">nil</span>, <span class="attribute">expires</span>=<span class="literal">nil</span>, <span class="attribute">httpOnly</span>=<span class="literal">nil</span>, <span class="attribute">secure</span>=<span class="literal">nil</span>&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>该方法的各个参数代表Cookie的各个属性。</p>
                  <p>示例如下：</p>
                  <figure class="highlight actionscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span><span class="params">(splash)</span></span></span><br><span class="line"><span class="function">    <span class="title">splash</span>:add_cookie</span>&#123;<span class="string">"sessionid"</span>, <span class="string">"237465ghgfsd"</span>, <span class="string">"/"</span>, domain=<span class="string">"http://example.com"</span>&#125;</span><br><span class="line">    splash:go(<span class="string">"http://example.com/"</span>)</span><br><span class="line">    <span class="keyword">return</span> splash:html()</span><br><span class="line">end</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="clear-cookies"><a href="#clear-cookies" class="headerlink" title="clear_cookies()"></a><code>clear_cookies()</code></h3>
                  <p>此方法可以清除所有的Cookies，示例如下：</p>
                  <figure class="highlight actionscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span><span class="params">(splash)</span></span></span><br><span class="line"><span class="function">    <span class="title">splash</span>:go<span class="params">(<span class="string">"https://www.baidu.com/"</span>)</span></span></span><br><span class="line"><span class="function">    <span class="title">splash</span>:clear_cookies<span class="params">()</span></span></span><br><span class="line"><span class="function">    <span class="title">return</span> <span class="title">splash</span>:get_cookies<span class="params">()</span></span></span><br><span class="line"><span class="function"><span class="title">end</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们清除了所有的Cookies，然后调用<code>get_cookies()</code>将结果返回。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight vbscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Splash <span class="built_in">Response</span>: <span class="built_in">Array</span>[<span class="number">0</span>]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，Cookies被全部清空，没有任何结果。</p>
                  <h3 id="get-viewport-size"><a href="#get-viewport-size" class="headerlink" title="get_viewport_size()"></a><code>get_viewport_size()</code></h3>
                  <p>此方法可以获取当前浏览器页面的大小，即宽高，示例如下：</p>
                  <figure class="highlight actionscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span><span class="params">(splash)</span></span></span><br><span class="line"><span class="function">    <span class="title">splash</span>:go<span class="params">(<span class="string">"https://www.baidu.com/"</span>)</span></span></span><br><span class="line"><span class="function">    <span class="title">return</span> <span class="title">splash</span>:get_viewport_size<span class="params">()</span></span></span><br><span class="line"><span class="function"><span class="title">end</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Splash Response: Array[<span class="number">2</span>]</span><br><span class="line"><span class="number">0</span>: <span class="number">1024</span></span><br><span class="line"><span class="number">1</span>: <span class="number">768</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="set-viewport-size"><a href="#set-viewport-size" class="headerlink" title="set_viewport_size()"></a><code>set_viewport_size()</code></h3>
                  <p>此方法可以设置当前浏览器页面的大小，即宽高，用法如下：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">splash:set<span class="constructor">_viewport_size(<span class="params">width</span>, <span class="params">height</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>例如，这里访问一个宽度自适应的页面：</p>
                  <figure class="highlight lua">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span><span class="params">(splash)</span></span></span><br><span class="line">    splash:set_viewport_size(<span class="number">400</span>, <span class="number">700</span>)</span><br><span class="line">    <span class="built_in">assert</span>(splash:go(<span class="string">"http://cuiqingcai.com"</span>))</span><br><span class="line">    <span class="keyword">return</span> splash:png()</span><br><span class="line"><span class="keyword">end</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如图7-14所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-14.jpg" alt="">图7-14 运行结果</p>
                  <h3 id="set-viewport-full"><a href="#set-viewport-full" class="headerlink" title="set_viewport_full()"></a><code>set_viewport_full()</code></h3>
                  <p>此方法可以设置浏览器全屏显示，示例如下：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">function</span> <span class="function"><span class="title">main</span>(<span class="variable">splash</span>)</span></span><br><span class="line">    <span class="variable">splash</span>:<span class="function"><span class="title">set_viewport_full</span>()</span></span><br><span class="line">    <span class="function"><span class="title"><span class="built_in">assert</span></span>(<span class="variable">splash</span>:<span class="title">go</span>(<span class="string">"http://cuiqingcai.com"</span>))</span></span><br><span class="line">    <span class="variable">return</span> <span class="variable">splash</span>:<span class="function"><span class="title">png</span>()</span></span><br><span class="line"><span class="variable">end</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="set-user-agent"><a href="#set-user-agent" class="headerlink" title="set_user_agent()"></a><code>set_user_agent()</code></h3>
                  <p>此方法可以设置浏览器的<code>User-Agent</code>，示例如下：</p>
                  <figure class="highlight actionscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span><span class="params">(splash)</span></span></span><br><span class="line"><span class="function">  <span class="title">splash</span>:set_user_agent<span class="params">(<span class="string">'Splash'</span>)</span></span></span><br><span class="line"><span class="function">  <span class="title">splash</span>:go<span class="params">(<span class="string">"http://httpbin.org/get"</span>)</span></span></span><br><span class="line"><span class="function">  <span class="title">return</span> <span class="title">splash</span>:html<span class="params">()</span></span></span><br><span class="line"><span class="function"><span class="title">end</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们将浏览器的<code>User-Agent</code>设置为<code>Splash</code>，运行结果如下：</p>
                  <figure class="highlight dts">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="params">&lt;html&gt;</span><span class="params">&lt;head&gt;</span><span class="params">&lt;/head&gt;</span><span class="params">&lt;body&gt;</span><span class="params">&lt;pre style="word-wrap: break-word; white-space: pre-wrap;"&gt;</span>&#123;</span><br><span class="line">  <span class="string">"args"</span>: &#123;&#125;, </span><br><span class="line">  <span class="string">"headers"</span>: &#123;</span><br><span class="line">    <span class="string">"Accept"</span>: <span class="string">"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"</span>, </span><br><span class="line">    <span class="string">"Accept-Encoding"</span>: <span class="string">"gzip, deflate"</span>, </span><br><span class="line">    <span class="string">"Accept-Language"</span>: <span class="string">"en,*"</span>, </span><br><span class="line">    <span class="string">"Connection"</span>: <span class="string">"close"</span>, </span><br><span class="line">    <span class="string">"Host"</span>: <span class="string">"httpbin.org"</span>, </span><br><span class="line">    <span class="string">"User-Agent"</span>: <span class="string">"Splash"</span></span><br><span class="line">  &#125;, </span><br><span class="line">  <span class="string">"origin"</span>: <span class="string">"60.207.237.85"</span>, </span><br><span class="line">  <span class="string">"url"</span>: <span class="string">"http://httpbin.org/get"</span></span><br><span class="line">&#125;</span><br><span class="line"><span class="params">&lt;/pre&gt;</span><span class="params">&lt;/body&gt;</span><span class="params">&lt;/html&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，此处<code>User-Agent</code>被成功设置。</p>
                  <h3 id="set-custom-headers"><a href="#set-custom-headers" class="headerlink" title="set_custom_headers()"></a><code>set_custom_headers()</code></h3>
                  <p>此方法可以设置请求头，示例如下：</p>
                  <figure class="highlight actionscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span><span class="params">(splash)</span></span></span><br><span class="line"><span class="function">  <span class="title">splash</span>:set_custom_headers<span class="params">(&#123;</span></span></span><br><span class="line"><span class="function"><span class="params">     [<span class="string">"User-Agent"</span>] = <span class="string">"Splash"</span>,</span></span></span><br><span class="line"><span class="function"><span class="params">     [<span class="string">"Site"</span>] = <span class="string">"Splash"</span>,</span></span></span><br><span class="line"><span class="function"><span class="params">  &#125;)</span></span></span><br><span class="line"><span class="function">  <span class="title">splash</span>:go<span class="params">(<span class="string">"http://httpbin.org/get"</span>)</span></span></span><br><span class="line"><span class="function">  <span class="title">return</span> <span class="title">splash</span>:html<span class="params">()</span></span></span><br><span class="line"><span class="function"><span class="title">end</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们设置了请求头中的<code>User-Agent</code>和<code>Site</code>属性，运行结果如下：</p>
                  <figure class="highlight dts">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="params">&lt;html&gt;</span><span class="params">&lt;head&gt;</span><span class="params">&lt;/head&gt;</span><span class="params">&lt;body&gt;</span><span class="params">&lt;pre style="word-wrap: break-word; white-space: pre-wrap;"&gt;</span>&#123;</span><br><span class="line">  <span class="string">"args"</span>: &#123;&#125;, </span><br><span class="line">  <span class="string">"headers"</span>: &#123;</span><br><span class="line">    <span class="string">"Accept"</span>: <span class="string">"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"</span>, </span><br><span class="line">    <span class="string">"Accept-Encoding"</span>: <span class="string">"gzip, deflate"</span>, </span><br><span class="line">    <span class="string">"Accept-Language"</span>: <span class="string">"en,*"</span>, </span><br><span class="line">    <span class="string">"Connection"</span>: <span class="string">"close"</span>, </span><br><span class="line">    <span class="string">"Host"</span>: <span class="string">"httpbin.org"</span>, </span><br><span class="line">    <span class="string">"Site"</span>: <span class="string">"Splash"</span>, </span><br><span class="line">    <span class="string">"User-Agent"</span>: <span class="string">"Splash"</span></span><br><span class="line">  &#125;, </span><br><span class="line">  <span class="string">"origin"</span>: <span class="string">"60.207.237.85"</span>, </span><br><span class="line">  <span class="string">"url"</span>: <span class="string">"http://httpbin.org/get"</span></span><br><span class="line">&#125;</span><br><span class="line"><span class="params">&lt;/pre&gt;</span><span class="params">&lt;/body&gt;</span><span class="params">&lt;/html&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="select"><a href="#select" class="headerlink" title="select()"></a><code>select()</code></h3>
                  <p>该方法可以选中符合条件的第一个节点，如果有多个节点符合条件，则只会返回一个，其参数是CSS选择器。示例如下：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">function</span> <span class="function"><span class="title">main</span>(<span class="variable">splash</span>)</span></span><br><span class="line">  <span class="variable">splash</span>:<span class="function"><span class="title">go</span>(<span class="string">"https://www.baidu.com/"</span>)</span></span><br><span class="line">  <span class="variable">input</span> = <span class="variable">splash</span>:<span class="function"><span class="title">select</span>(<span class="string">"#kw"</span>)</span></span><br><span class="line">  <span class="variable">input</span>:<span class="function"><span class="title">send_text</span>(<span class="string">'Splash'</span>)</span></span><br><span class="line">  <span class="variable">splash</span>:<span class="function"><span class="title">wait</span>(<span class="number">3</span>)</span></span><br><span class="line">  <span class="variable">return</span> <span class="variable">splash</span>:<span class="function"><span class="title">png</span>()</span></span><br><span class="line"><span class="variable">end</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们首先访问了百度，然后选中了搜索框，随后调用了<code>send_text()</code>方法填写了文本，然后返回网页截图。</p>
                  <p>结果如图7-15所示，可以看到，我们成功填写了输入框。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-15.jpg" alt="">图7-15 运行结果</p>
                  <h3 id="select-all"><a href="#select-all" class="headerlink" title="select_all()"></a><code>select_all()</code></h3>
                  <p>此方法可以选中所有符合条件的节点，其参数是CSS选择器。示例如下：</p>
                  <figure class="highlight lua">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">main</span><span class="params">(splash)</span></span></span><br><span class="line">  <span class="keyword">local</span> treat = <span class="built_in">require</span>(<span class="string">'treat'</span>)</span><br><span class="line">  <span class="built_in">assert</span>(splash:go(<span class="string">"http://quotes.toscrape.com/"</span>))</span><br><span class="line">  <span class="built_in">assert</span>(splash:wait(<span class="number">0.5</span>))</span><br><span class="line">  <span class="keyword">local</span> texts = splash:select_all(<span class="string">'.quote .text'</span>)</span><br><span class="line">  <span class="keyword">local</span> results = &#123;&#125;</span><br><span class="line">  <span class="keyword">for</span> index, text <span class="keyword">in</span> <span class="built_in">ipairs</span>(texts) <span class="keyword">do</span></span><br><span class="line">    results[index] = text.node.innerHTML</span><br><span class="line">  <span class="keyword">end</span></span><br><span class="line">  <span class="keyword">return</span> treat.as_array(results)</span><br><span class="line"><span class="keyword">end</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们通过CSS选择器选中了节点的正文内容，随后遍历了所有节点，将其中的文本获取下来。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight smalltalk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="type">Splash</span> <span class="type">Response</span>: <span class="type">Array</span>[<span class="number">10</span>]</span><br><span class="line"><span class="number">0</span>: <span class="comment">"“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”"</span></span><br><span class="line"><span class="number">1</span>: <span class="comment">"“It is our choices, Harry, that show what we truly are, far more than our abilities.”"</span></span><br><span class="line"><span class="number">2</span>: “<span class="type">There</span> are only two ways to live your life. <span class="type">One</span> is as though nothing is a miracle. <span class="type">The</span> other is as though everything is a miracle.”</span><br><span class="line"><span class="number">3</span>: <span class="comment">"“The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.”"</span></span><br><span class="line"><span class="number">4</span>: <span class="comment">"“Imperfection is beauty, madness is genius and it's better to be absolutely ridiculous than absolutely boring.”"</span></span><br><span class="line"><span class="number">5</span>: <span class="comment">"“Try not to become a man of success. Rather become a man of value.”"</span></span><br><span class="line"><span class="number">6</span>: <span class="comment">"“It is better to be hated for what you are than to be loved for what you are not.”"</span></span><br><span class="line"><span class="number">7</span>: <span class="comment">"“I have not failed. I've just found 10,000 ways that won't work.”"</span></span><br><span class="line"><span class="number">8</span>: <span class="comment">"“A woman is like a tea bag; you never know how strong it is until it's in hot water.”"</span></span><br><span class="line"><span class="number">9</span>: <span class="comment">"“A day without sunshine is like, you know, night.”"</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以发现，我们成功地将10个节点的正文内容获取了下来。</p>
                  <h3 id="mouse-click"><a href="#mouse-click" class="headerlink" title="mouse_click()"></a><code>mouse_click()</code></h3>
                  <p>此方法可以模拟鼠标点击操作，传入的参数为坐标值<code>x</code>和<code>y</code>。此外，也可以直接选中某个节点，然后调用此方法，示例如下：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">function</span> <span class="function"><span class="title">main</span>(<span class="variable">splash</span>)</span></span><br><span class="line">  <span class="variable">splash</span>:<span class="function"><span class="title">go</span>(<span class="string">"https://www.baidu.com/"</span>)</span></span><br><span class="line">  <span class="variable">input</span> = <span class="variable">splash</span>:<span class="function"><span class="title">select</span>(<span class="string">"#kw"</span>)</span></span><br><span class="line">  <span class="variable">input</span>:<span class="function"><span class="title">send_text</span>(<span class="string">'Splash'</span>)</span></span><br><span class="line">  <span class="variable">submit</span> = <span class="variable">splash</span>:<span class="function"><span class="title">select</span>(<span class="string">'#su'</span>)</span></span><br><span class="line">  <span class="variable">submit</span>:<span class="function"><span class="title">mouse_click</span>()</span></span><br><span class="line">  <span class="variable">splash</span>:<span class="function"><span class="title">wait</span>(<span class="number">3</span>)</span></span><br><span class="line">  <span class="variable">return</span> <span class="variable">splash</span>:<span class="function"><span class="title">png</span>()</span></span><br><span class="line"><span class="variable">end</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们首先选中页面的输入框，输入了文本，然后选中“提交”按钮，调用了<code>mouse_click()</code>方法提交查询，然后页面等待三秒，返回截图，结果如图7-16所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-16.jpg" alt="">图7-16 运行结果</p>
                  <p>可以看到，这里我们成功获取了查询后的页面内容，模拟了百度搜索操作。</p>
                  <p>前面介绍了Splash的常用API操作，还有一些API在这不再一一介绍，更加详细和权威的说明可以参见官方文档<a href="https://splash.readthedocs.io/en/stable/scripting-ref.html" target="_blank" rel="noopener">https://splash.readthedocs.io/en/stable/scripting-ref.html</a>，此页面介绍了Splash对象的所有API操作。另外，还有针对页面元素的API操作，链接为<a href="https://splash.readthedocs.io/en/stable/scripting-element-object.html" target="_blank" rel="noopener">https://splash.readthedocs.io/en/stable/scripting-element-object.html</a>。</p>
                  <h2 id="7-Splash-API调用"><a href="#7-Splash-API调用" class="headerlink" title="7. Splash API调用"></a>7. Splash API调用</h2>
                  <p>前面说明了Splash Lua脚本的用法，但这些脚本是在Splash页面中测试运行的，如何才能利用Splash渲染页面呢？怎样才能和Python程序结合使用并抓取JavaScript渲染的页面呢？</p>
                  <p>其实Splash给我们提供了一些HTTP API接口，我们只需要请求这些接口并传递相应的参数即可，下面简要介绍这些接口。</p>
                  <h3 id="render-html"><a href="#render-html" class="headerlink" title="render.html"></a>render.html</h3>
                  <p>此接口用于获取JavaScript渲染的页面的HTML代码，接口地址就是Splash的运行地址加此接口名称，例如<a href="http://localhost:8050/render.html" target="_blank" rel="noopener">http://localhost:8050/render.html</a>。可以用<code>curl</code>来测试一下：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">curl http://localhos<span class="variable">t:8050</span>/render.html?url=http<span class="variable">s:</span>//www.baidu.<span class="keyword">com</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们给此接口传递了一个<code>url</code>参数来指定渲染的URL，返回结果即页面渲染后的源代码。</p>
                  <p>如果用Python实现的话，代码如下：</p>
                  <figure class="highlight processing">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line">url = <span class="string">'http://localhost:8050/render.html?url=https://www.baidu.com'</span></span><br><span class="line">response = requests.<span class="built_in">get</span>(url)</span><br><span class="line"><span class="built_in">print</span>(response.<span class="built_in">text</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就可以成功输出百度页面渲染后的源代码了。</p>
                  <p>另外，此接口还可以指定其他参数，比如通过<code>wait</code>指定等待秒数。如果要确保页面完全加载出来，可以增加等待时间，例如：</p>
                  <figure class="highlight processing">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line">url = <span class="string">'http://localhost:8050/render.html?url=https://www.taobao.com&amp;wait=5'</span></span><br><span class="line">response = requests.<span class="built_in">get</span>(url)</span><br><span class="line"><span class="built_in">print</span>(response.<span class="built_in">text</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>此时得到响应的时间就会相应变长，比如这里会等待5秒多钟才能获取淘宝页面的源代码。</p>
                  <p>另外，此接口还支持代理设置、图片加载设置、Headers设置、请求方法设置，具体的用法可以参见官方文档<a href="https://splash.readthedocs.io/en/stable/api.html#render-html" target="_blank" rel="noopener">https://splash.readthedocs.io/en/stable/api.html#render-html</a>。</p>
                  <h3 id="render-png"><a href="#render-png" class="headerlink" title="render.png"></a>render.png</h3>
                  <p>此接口可以获取网页截图，其参数比render.html多了几个，比如通过<code>width</code>和<code>height</code>来控制宽高，它返回的是PNG格式的图片二进制数据。示例如下：</p>
                  <figure class="highlight sas">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">curl http://localhost:8050/render.png?url=https://www.taobao.com<span class="variable">&amp;wait</span>=5<span class="variable">&amp;width</span>=1000<span class="variable">&amp;height</span>=700</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们传入了<code>width</code>和<code>height</code>来设置页面大小为1000×700像素。</p>
                  <p>如果用Python实现，可以将返回的二进制数据保存为PNG格式的图片，具体如下：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line"></span><br><span class="line">url = <span class="string">'http://localhost:8050/render.png?url=https://www.jd.com&amp;wait=5&amp;width=1000&amp;height=700'</span></span><br><span class="line">response = requests.<span class="built_in">get</span>(url)</span><br><span class="line"><span class="keyword">with</span> <span class="built_in">open</span>(<span class="string">'taobao.png'</span>, <span class="string">'wb'</span>) <span class="keyword">as</span> f:</span><br><span class="line">    f.<span class="built_in">write</span>(response.content)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>得到的图片如图7-17所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-17.png" alt="">图7-17 运行结果</p>
                  <p>这样我们就成功获取了京东首页渲染完成后的页面截图，详细的参数设置可以参考官网文档<a href="https://splash.readthedocs.io/en/stable/api.html#render-png" target="_blank" rel="noopener">https://splash.readthedocs.io/en/stable/api.html#render-png</a>。</p>
                  <h3 id="render-jpeg"><a href="#render-jpeg" class="headerlink" title="render.jpeg"></a>render.jpeg</h3>
                  <p>此接口和render.png类似，不过它返回的是JPEG格式的图片二进制数据。</p>
                  <p>另外，此接口比render.png多了参数<code>quality</code>，它用来设置图片质量。</p>
                  <h3 id="render-har"><a href="#render-har" class="headerlink" title="render.har"></a>render.har</h3>
                  <p>此接口用于获取页面加载的HAR数据，示例如下：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">curl http://localhos<span class="variable">t:8050</span>/render.har?url=http<span class="variable">s:</span>//www.jd.<span class="keyword">com</span>&amp;wait=<span class="number">5</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>它的返回结果（如图7-18所示）非常多，是一个JSON格式的数据，其中包含页面加载过程中的HAR数据。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-18.jpg" alt="">图7-18 运行结果</p>
                  <h3 id="render-json"><a href="#render-json" class="headerlink" title="render.json"></a>render.json</h3>
                  <p>此接口包含了前面接口的所有功能，返回结果是JSON格式，示例如下：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">curl http:<span class="regexp">//</span>localhost:<span class="number">8050</span><span class="regexp">/render.json?url=https:/</span><span class="regexp">/httpbin.org</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;<span class="attr">"title"</span>: <span class="string">"httpbin(1): HTTP Client Testing Service"</span>, <span class="attr">"url"</span>: <span class="string">"https://httpbin.org/"</span>, <span class="attr">"requestedUrl"</span>: <span class="string">"https://httpbin.org/"</span>, <span class="attr">"geometry"</span>: [<span class="number">0</span>, <span class="number">0</span>, <span class="number">1024</span>, <span class="number">768</span>]&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，这里以JSON形式返回了相应的请求数据。</p>
                  <p>我们可以通过传入不同参数控制其返回结果。比如，传入<code>html=1</code>，返回结果即会增加源代码数据；传入<code>png=1</code>，返回结果即会增加页面PNG截图数据；传入<code>har=1</code>，则会获得页面HAR数据。例如：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">curl http:<span class="regexp">//</span>localhost:<span class="number">8050</span><span class="regexp">/render.json?url=https:/</span><span class="regexp">/httpbin.org&amp;html=1&amp;har=1</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样返回的JSON结果会包含网页源代码和HAR数据。</p>
                  <p>此外还有更多参数设置，具体可以参考官方文档：<a href="https://splash.readthedocs.io/en/stable/api.html#render-json" target="_blank" rel="noopener">https://splash.readthedocs.io/en/stable/api.html#render-json</a>。</p>
                  <h3 id="execute"><a href="#execute" class="headerlink" title="execute"></a>execute</h3>
                  <p>此接口才是最为强大的接口。前面说了很多Splash Lua脚本的操作，用此接口便可实现与Lua脚本的对接。</p>
                  <p>前面的render.html和render.png等接口对于一般的JavaScript渲染页面是足够了，但是如果要实现一些交互操作的话，它们还是无能为力，这里就需要使用execute接口了。</p>
                  <p>我们先实现一个最简单的脚本，直接返回数据：</p>
                  <figure class="highlight ada">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">function</span> <span class="title">main</span>(splash)</span><br><span class="line">    <span class="keyword">return</span> <span class="type">'hello'</span></span><br><span class="line"><span class="keyword">end</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后将此脚本转化为URL编码后的字符串，拼接到execute接口后面，示例如下：</p>
                  <figure class="highlight xquery">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">curl http://localhost:<span class="number">8050</span>/execute?lua_source=<span class="keyword">function</span>+main<span class="meta">%28splash</span><span class="meta">%29</span><span class="meta">%0D</span><span class="meta">%0A</span>++<span class="keyword">return</span>+<span class="meta">%27hello</span><span class="meta">%27</span><span class="meta">%0D</span><span class="meta">%0Aend</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">hello</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们通过<code>lua_source</code>参数传递了转码后的Lua脚本，通过execute接口获取了最终脚本的执行结果。</p>
                  <p>这里我们更加关心的肯定是如何用Python来实现，上例用Python实现的话，代码如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">from</span> urllib.parse <span class="keyword">import</span> quote</span><br><span class="line"></span><br><span class="line">lua = <span class="string">'''</span></span><br><span class="line"><span class="string">function main(splash)</span></span><br><span class="line"><span class="string">    return 'hello'</span></span><br><span class="line"><span class="string">end</span></span><br><span class="line"><span class="string">'''</span></span><br><span class="line"></span><br><span class="line">url = <span class="string">'http://localhost:8050/execute?lua_source='</span> + quote(lua)</span><br><span class="line">response = requests.get(url)</span><br><span class="line">print(response.text)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">hello</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们用Python中的三引号将Lua脚本包括起来，然后用urllib.parse模块里的<code>quote()</code>方法将脚本进行URL转码，随后构造了Splash请求URL，将其作为<code>lua_source</code>参数传递，这样运行结果就会显示Lua脚本执行后的结果。</p>
                  <p>我们再通过实例看一下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">from</span> urllib.parse <span class="keyword">import</span> quote</span><br><span class="line"></span><br><span class="line">lua = <span class="string">'''</span></span><br><span class="line"><span class="string">function main(splash, args)</span></span><br><span class="line"><span class="string">  local treat = require("treat")</span></span><br><span class="line"><span class="string">  local response = splash:http_get("http://httpbin.org/get")</span></span><br><span class="line"><span class="string">    return &#123;</span></span><br><span class="line"><span class="string">    html=treat.as_string(response.body),</span></span><br><span class="line"><span class="string">    url=response.url,</span></span><br><span class="line"><span class="string">    status=response.status</span></span><br><span class="line"><span class="string">    &#125;</span></span><br><span class="line"><span class="string">end</span></span><br><span class="line"><span class="string">'''</span></span><br><span class="line"></span><br><span class="line">url = <span class="string">'http://localhost:8050/execute?lua_source='</span> + quote(lua)</span><br><span class="line">response = requests.get(url)</span><br><span class="line">print(response.text)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight taggerscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;"url": "http://httpbin.org/get", "status": 200, "html": "&#123;<span class="symbol">\n</span>  <span class="symbol">\"</span>args<span class="symbol">\"</span>: &#123;&#125;, <span class="symbol">\n</span>  <span class="symbol">\"</span>headers<span class="symbol">\"</span>: &#123;<span class="symbol">\n</span>    <span class="symbol">\"</span>Accept-Encoding<span class="symbol">\"</span>: <span class="symbol">\"</span>gzip, deflate<span class="symbol">\"</span>, <span class="symbol">\n</span>    <span class="symbol">\"</span>Accept-Language<span class="symbol">\"</span>: <span class="symbol">\"</span>en,*<span class="symbol">\"</span>, <span class="symbol">\n</span>    <span class="symbol">\"</span>Connection<span class="symbol">\"</span>: <span class="symbol">\"</span>close<span class="symbol">\"</span>, <span class="symbol">\n</span>    <span class="symbol">\"</span>Host<span class="symbol">\"</span>: <span class="symbol">\"</span>httpbin.org<span class="symbol">\"</span>, <span class="symbol">\n</span>    <span class="symbol">\"</span>User-Agent<span class="symbol">\"</span>: <span class="symbol">\"</span>Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/602.1 (KHTML, like Gecko) splash Version/9.0 Safari/602.1<span class="symbol">\"</span><span class="symbol">\n</span>  &#125;, <span class="symbol">\n</span>  <span class="symbol">\"</span>origin<span class="symbol">\"</span>: <span class="symbol">\"</span>60.207.237.85<span class="symbol">\"</span>, <span class="symbol">\n</span>  <span class="symbol">\"</span>url<span class="symbol">\"</span>: <span class="symbol">\"</span>http://httpbin.org/get<span class="symbol">\"</span><span class="symbol">\n</span>&#125;<span class="symbol">\n</span>"&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，返回结果是JSON形式，我们成功获取了请求的URL、状态码和网页源代码。</p>
                  <p>如此一来，我们之前所说的Lua脚本均可以用此方式与Python进行对接，所有网页的动态渲染、模拟点击、表单提交、页面滑动、延时等待后的一些结果均可以自由控制，获取页面源码和截图也都不在话下。</p>
                  <p>到现在为止，我们可以用Python和Splash实现JavaScript渲染的页面的抓取了。除了Selenium，本节所说的Splash同样可以做到非常强大的渲染功能，同时它也不需要浏览器即可渲染，使用非常方便。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-31 20:41:33" itemprop="dateCreated datePublished" datetime="2018-01-31T20:41:33+08:00">2018-01-31</time>
                </span>
                <span id="/5638.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 7.2-Splash的使用" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>22k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>20 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5630.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5630.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 7.1-Selenium的使用</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>Selenium是一个自动化测试工具，利用它可以驱动浏览器执行特定的动作，如点击、下拉等操作，同时还可以获取浏览器当前呈现的页面的源代码，做到可见即可爬。对于一些JavaScript动态渲染的页面来说，此种抓取方式非常有效。本节中，就让我们来感受一下它的强大之处吧。</p>
                  <h2 id="1-准备工作"><a href="#1-准备工作" class="headerlink" title="1. 准备工作"></a>1. 准备工作</h2>
                  <p>本节以Chrome为例来讲解Selenium的用法。在开始之前，请确保已经正确安装好了Chrome浏览器并配置好了ChromeDriver。另外，还需要正确安装好Python的Selenium库，详细的安装和配置过程可以参考第1章。</p>
                  <h2 id="2-基本使用"><a href="#2-基本使用" class="headerlink" title="2. 基本使用"></a>2. 基本使用</h2>
                  <p>准备工作做好之后，首先来大体看一下Selenium有一些怎样的功能。示例如下：</p>
                  <figure class="highlight sas">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta">from</span> selenium import webdriver</span><br><span class="line"><span class="meta">from</span> selenium.webdriver.common.<span class="meta">by</span> import <span class="meta">By</span></span><br><span class="line"><span class="meta">from</span> selenium.webdriver.common.keys import Keys</span><br><span class="line"><span class="meta">from</span> selenium.webdriver.support import expected_conditions <span class="meta">as</span> EC</span><br><span class="line"><span class="meta">from</span> selenium.webdriver.support.wait import WebDriverWait</span><br><span class="line"></span><br><span class="line">browser = webdriver.Chrome()</span><br><span class="line">try:</span><br><span class="line">    browser.get(<span class="string">'https://www.baidu.com'</span>)</span><br><span class="line">    <span class="meta">input</span> = browser.find_element_by_id(<span class="string">'kw'</span>)</span><br><span class="line">    <span class="meta">input</span>.send_keys(<span class="string">'Python'</span>)</span><br><span class="line">    <span class="meta">input</span>.send_keys(Keys.ENTER)</span><br><span class="line">    wait = WebDriverWait(browser, 10)</span><br><span class="line">    wait.<span class="meta">until</span>(EC.presence_of_element_located((<span class="meta">By</span>.ID, <span class="string">'content_left'</span>)))</span><br><span class="line">    p<span class="meta">rint(</span>browser.current_url)</span><br><span class="line">    p<span class="meta">rint(</span>browser.get_cookies())</span><br><span class="line">    p<span class="meta">rint(</span>browser.page_source)</span><br><span class="line">finally:</span><br><span class="line">    browser<span class="meta">.close(</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行代码后发现，会自动弹出一个Chrome浏览器。浏览器首先会跳转到百度，然后在搜索框中输入Python，接着跳转到搜索结果页，如图7-1所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-1.png" alt="">图7-1 运行结果</p>
                  <p>搜索结果加载出来后，控制台分别会输出当前的URL、当前的Cookies和网页源代码：</p>
                  <figure class="highlight sas">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">https://www.baidu.com/s?ie=utf-8<span class="variable">&amp;f</span>=8<span class="variable">&amp;rsv_bp</span>=0<span class="variable">&amp;rsv_idx</span>=1<span class="variable">&amp;tn</span>=baidu<span class="variable">&amp;wd</span>=Python<span class="variable">&amp;rsv_pq</span>=c94d0df9000a72d0<span class="variable">&amp;rsv_t</span>=07099xvun1ZmC0bf6eQvygJ43IUTTUOl5FCJVPgwG2YREs70GplJjH2F%2BCQ<span class="variable">&amp;rqlang</span>=cn<span class="variable">&amp;rsv_enter</span>=1<span class="variable">&amp;rsv_sug3</span>=6<span class="variable">&amp;rsv_sug2</span>=0<span class="variable">&amp;inputT</span>=87<span class="variable">&amp;rsv_sug4</span>=87</span><br><span class="line">[&#123;<span class="string">'secure'</span>: False, <span class="string">'value'</span>: <span class="string">'B490B5EBF6F3CD402E515D22BCDA1598'</span>, <span class="string">'domain'</span>: <span class="string">'.baidu.com'</span>, <span class="string">'path'</span>: <span class="string">'/'</span>, <span class="string">'httpOnly'</span>: False, <span class="string">'name'</span>: <span class="string">'BDORZ'</span>, <span class="string">'expiry'</span>: 1491688071.707553&#125;, &#123;<span class="string">'secure'</span>: False, <span class="string">'value'</span>: <span class="string">'22473_1441_21084_17001'</span>, <span class="string">'domain'</span>: <span class="string">'.baidu.com'</span>, <span class="string">'path'</span>: <span class="string">'/'</span>, <span class="string">'httpOnly'</span>: False, <span class="string">'name'</span>: <span class="string">'H_PS_PSSID'</span>&#125;, &#123;<span class="string">'secure'</span>: False, <span class="string">'value'</span>: <span class="string">'12883875381399993259_00_0_I_R_2_0303_C02F_N_I_I_0'</span>, <span class="string">'domain'</span>: <span class="string">'.www.baidu.com'</span>, <span class="string">'path'</span>: <span class="string">'/'</span>, <span class="string">'httpOnly'</span>: False, <span class="string">'name'</span>: <span class="string">'__bsi'</span>, <span class="string">'expiry'</span>: 1491601676.69722&#125;]</span><br><span class="line">&lt;!DOCTYPE html&gt;&lt;!--STATUS OK--&gt;...&lt;/html&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>源代码过长，在此省略。可以看到，我们得到的当前URL、Cookies和源代码都是浏览器中的真实内容。</p>
                  <p>所以说，如果用Selenium来驱动浏览器加载网页的话，就可以直接拿到JavaScript渲染的结果了，不用担心使用的是什么加密系统。</p>
                  <p>下面来详细了解一下Selenium的用法。</p>
                  <h2 id="3-声明浏览器对象"><a href="#3-声明浏览器对象" class="headerlink" title="3. 声明浏览器对象"></a>3. 声明浏览器对象</h2>
                  <p>Selenium支持非常多的浏览器，如Chrome、Firefox、Edge等，还有Android、BlackBerry等手机端的浏览器。另外，也支持无界面浏览器PhantomJS。</p>
                  <p>此外，我们可以用如下方式初始化：</p>
                  <figure class="highlight armasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">from</span> <span class="keyword">selenium </span><span class="meta">import</span> webdriver</span><br><span class="line"></span><br><span class="line"><span class="keyword">browser </span>= webdriver.Chrome()</span><br><span class="line"><span class="keyword">browser </span>= webdriver.Firefox()</span><br><span class="line"><span class="keyword">browser </span>= webdriver.Edge()</span><br><span class="line"><span class="keyword">browser </span>= webdriver.PhantomJS()</span><br><span class="line"><span class="keyword">browser </span>= webdriver.Safari()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就完成了浏览器对象的初始化并将其赋值为<code>browser</code>对象。接下来，我们要做的就是调用<code>browser</code>对象，让其执行各个动作以模拟浏览器操作。</p>
                  <h2 id="4-访问页面"><a href="#4-访问页面" class="headerlink" title="4. 访问页面"></a>4. 访问页面</h2>
                  <p>我们可以用<code>get()</code>方法来请求网页，参数传入链接URL即可。比如，这里用<code>get()</code>方法访问淘宝，然后打印出源代码，代码如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> selenium import webdriver</span><br><span class="line"></span><br><span class="line">browser = webdriver.Chrome()</span><br><span class="line">browser.<span class="builtin-name">get</span>(<span class="string">'https://www.taobao.com'</span>)</span><br><span class="line"><span class="builtin-name">print</span>(browser.page_source)</span><br><span class="line">browser.close()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行后发现，弹出了Chrome浏览器并且自动访问了淘宝，然后控制台输出了淘宝页面的源代码，随后浏览器关闭。</p>
                  <p>通过这几行简单的代码，我们可以实现浏览器的驱动并获取网页源码，非常便捷。</p>
                  <h2 id="5-查找节点"><a href="#5-查找节点" class="headerlink" title="5. 查找节点"></a>5. 查找节点</h2>
                  <p>Selenium可以驱动浏览器完成各种操作，比如填充表单、模拟点击等。比如，我们想要完成向某个输入框输入文字的操作，总需要知道这个输入框在哪里吧？而Selenium提供了一系列查找节点的方法，我们可以用这些方法来获取想要的节点，以便下一步执行一些动作或者提取信息。</p>
                  <h3 id="单个节点"><a href="#单个节点" class="headerlink" title="单个节点"></a>单个节点</h3>
                  <p>比如，想要从淘宝页面中提取搜索框这个节点，首先要观察它的源代码，如图7-2所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-2.png" alt="">图7-2 源代码</p>
                  <p>可以发现，它的<code>id</code>是<code>q</code>，<code>name</code>也是<code>q</code>。此外，还有许多其他属性，此时我们就可以用多种方式获取它了。比如，<code>find_element_by_name()</code>是根据<code>name</code>值获取，<code>find_element_by_id()</code>是根据<code>id</code>获取。另外，还有根据XPath、CSS选择器等获取的方式。</p>
                  <p>我们用代码实现一下：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from selenium import webdriver</span><br><span class="line"></span><br><span class="line">browser = webdriver.<span class="constructor">Chrome()</span></span><br><span class="line">browser.get('https:<span class="comment">//www.taobao.com')</span></span><br><span class="line">input_first = browser.find<span class="constructor">_element_by_id('<span class="params">q</span>')</span></span><br><span class="line">input_second = browser.find<span class="constructor">_element_by_css_selector('#<span class="params">q</span>')</span></span><br><span class="line">input_third = browser.find<span class="constructor">_element_by_xpath('<span class="operator">/</span><span class="operator">/</span><span class="operator">*</span>[@<span class="params">id</span>=<span class="string">"q"</span>]')</span></span><br><span class="line">print(input_first, input_second, input_third)</span><br><span class="line">browser.close<span class="literal">()</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们使用3种方式获取输入框，分别是根据ID、CSS选择器和XPath获取，它们返回的结果完全一致。运行结果如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;selenium.webdriver.remote.webelement.WebElement (<span class="attribute">session</span>=<span class="string">"5e53d9e1c8646e44c14c1c2880d424af"</span>, <span class="attribute">element</span>=<span class="string">"0.5649563096161541-1"</span>)&gt; </span><br><span class="line">&lt;selenium.webdriver.remote.webelement.WebElement (<span class="attribute">session</span>=<span class="string">"5e53d9e1c8646e44c14c1c2880d424af"</span>, <span class="attribute">element</span>=<span class="string">"0.5649563096161541-1"</span>)&gt; </span><br><span class="line">&lt;selenium.webdriver.remote.webelement.WebElement (<span class="attribute">session</span>=<span class="string">"5e53d9e1c8646e44c14c1c2880d424af"</span>, <span class="attribute">element</span>=<span class="string">"0.5649563096161541-1"</span>)&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，这3个节点都是<code>WebElement</code>类型，是完全一致的。</p>
                  <p>这里列出所有获取单个节点的方法：</p>
                  <figure class="highlight ceylon">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">find<span class="number">_</span>element<span class="number">_</span><span class="meta">by</span><span class="number">_</span>id</span><br><span class="line">find<span class="number">_</span>element<span class="number">_</span><span class="meta">by</span><span class="number">_n</span>ame</span><br><span class="line">find<span class="number">_</span>element<span class="number">_</span><span class="meta">by</span><span class="number">_</span>xpath</span><br><span class="line">find<span class="number">_</span>element<span class="number">_</span><span class="meta">by</span><span class="number">_</span>link<span class="number">_</span>text</span><br><span class="line">find<span class="number">_</span>element<span class="number">_</span><span class="meta">by</span><span class="number">_p</span>artial<span class="number">_</span>link<span class="number">_</span>text</span><br><span class="line">find<span class="number">_</span>element<span class="number">_</span><span class="meta">by</span><span class="number">_</span>tag<span class="number">_n</span>ame</span><br><span class="line">find<span class="number">_</span>element<span class="number">_</span><span class="meta">by</span><span class="number">_</span><span class="keyword">class</span><span class="number">_n</span>ame</span><br><span class="line">find<span class="number">_</span>element<span class="number">_</span><span class="meta">by</span><span class="number">_</span>css<span class="number">_</span>selector</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外，Selenium还提供了通用方法<code>find_element()</code>，它需要传入两个参数：查找方式<code>By</code>和值。实际上，它就是<code>find_element_by_id()</code>这种方法的通用函数版本，比如<code>find_element_by_id(id)</code>就等价于<code>find_element(By.ID, id)</code>，二者得到的结果完全一致。我们用代码实现一下：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> selenium <span class="keyword">import</span> webdriver</span><br><span class="line"><span class="keyword">from</span> selenium.webdriver.common.<span class="keyword">by</span> <span class="keyword">import</span> <span class="keyword">By</span></span><br><span class="line"></span><br><span class="line">browser = webdriver.Chrome()</span><br><span class="line">browser.<span class="keyword">get</span>(<span class="string">'https://www.taobao.com'</span>)</span><br><span class="line">input_first = browser.find_element(<span class="keyword">By</span>.ID, <span class="string">'q'</span>)</span><br><span class="line">print(input_first)</span><br><span class="line">browser.<span class="keyword">close</span>()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>实际上，这种查找方式的功能和上面列举的查找函数完全一致，不过参数更加灵活。</p>
                  <h3 id="多个节点"><a href="#多个节点" class="headerlink" title="多个节点"></a>多个节点</h3>
                  <p>如果查找的目标在网页中只有一个，那么完全可以用<code>find_element()</code>方法。但如果有多个节点，再用<code>find_element()</code>方法查找，就只能得到第一个节点了。如果要查找所有满足条件的节点，需要用<code>find_elements()</code>这样的方法。注意，在这个方法的名称中，element多了一个s，注意区分。</p>
                  <p>比如，要查找淘宝左侧导航条的所有条目，如图7-3所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-3.jpg" alt="">图7-3 导航栏</p>
                  <p>就可以这样来实现：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> selenium import webdriver</span><br><span class="line"></span><br><span class="line">browser = webdriver.Chrome()</span><br><span class="line">browser.<span class="builtin-name">get</span>(<span class="string">'https://www.taobao.com'</span>)</span><br><span class="line">lis = browser.find_elements_by_css_selector(<span class="string">'.service-bd li'</span>)</span><br><span class="line"><span class="builtin-name">print</span>(lis)</span><br><span class="line">browser.close()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[&lt;selenium<span class="selector-class">.webdriver</span><span class="selector-class">.remote</span><span class="selector-class">.webelement</span><span class="selector-class">.WebElement</span> (session=<span class="string">"c26290835d4457ebf7d96bfab3740d19"</span>, element=<span class="string">"0.09221044033125603-1"</span>)&gt;, &lt;selenium<span class="selector-class">.webdriver</span><span class="selector-class">.remote</span><span class="selector-class">.webelement</span><span class="selector-class">.WebElement</span> (session=<span class="string">"c26290835d4457ebf7d96bfab3740d19"</span>, element=<span class="string">"0.09221044033125603-2"</span>)&gt;, &lt;selenium<span class="selector-class">.webdriver</span><span class="selector-class">.remote</span><span class="selector-class">.webelement</span><span class="selector-class">.WebElement</span> (session=<span class="string">"c26290835d4457ebf7d96bfab3740d19"</span>, element=<span class="string">"0.09221044033125603-3"</span>)&gt;...&lt;selenium<span class="selector-class">.webdriver</span><span class="selector-class">.remote</span><span class="selector-class">.webelement</span><span class="selector-class">.WebElement</span> (session=<span class="string">"c26290835d4457ebf7d96bfab3740d19"</span>, element=<span class="string">"0.09221044033125603-16"</span>)&gt;]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里简化了输出结果，中间部分省略。</p>
                  <p>可以看到，得到的内容变成了列表类型，列表中的每个节点都是<code>WebElement</code>类型。</p>
                  <p>也就是说，如果我们用<code>find_element()</code>方法，只能获取匹配的第一个节点，结果是<code>WebElement</code>类型。如果用<code>find_elements()</code>方法，则结果是列表类型，列表中的每个节点是<code>WebElement</code>类型。</p>
                  <p>这里列出所有获取多个节点的方法：</p>
                  <figure class="highlight ceylon">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">find<span class="number">_</span>elements<span class="number">_</span><span class="meta">by</span><span class="number">_</span>id</span><br><span class="line">find<span class="number">_</span>elements<span class="number">_</span><span class="meta">by</span><span class="number">_n</span>ame</span><br><span class="line">find<span class="number">_</span>elements<span class="number">_</span><span class="meta">by</span><span class="number">_</span>xpath</span><br><span class="line">find<span class="number">_</span>elements<span class="number">_</span><span class="meta">by</span><span class="number">_</span>link<span class="number">_</span>text</span><br><span class="line">find<span class="number">_</span>elements<span class="number">_</span><span class="meta">by</span><span class="number">_p</span>artial<span class="number">_</span>link<span class="number">_</span>text</span><br><span class="line">find<span class="number">_</span>elements<span class="number">_</span><span class="meta">by</span><span class="number">_</span>tag<span class="number">_n</span>ame</span><br><span class="line">find<span class="number">_</span>elements<span class="number">_</span><span class="meta">by</span><span class="number">_</span><span class="keyword">class</span><span class="number">_n</span>ame</span><br><span class="line">find<span class="number">_</span>elements<span class="number">_</span><span class="meta">by</span><span class="number">_</span>css<span class="number">_</span>selector</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>当然，我们也可以直接用<code>find_elements()</code>方法来选择，这时可以这样写：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">lis = browser.find<span class="constructor">_elements(By.CSS_SELECTOR, '.<span class="params">service</span>-<span class="params">bd</span> <span class="params">li</span>')</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果是完全一致的。</p>
                  <h2 id="6-节点交互"><a href="#6-节点交互" class="headerlink" title="6. 节点交互"></a>6. 节点交互</h2>
                  <p>Selenium可以驱动浏览器来执行一些操作，也就是说可以让浏览器模拟执行一些动作。比较常见的用法有：输入文字时用<code>send_keys()</code>方法，清空文字时用<code>clear()</code>方法，点击按钮时用<code>click()</code>方法。示例如下：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> selenium <span class="keyword">import</span> webdriver</span><br><span class="line"><span class="keyword">import</span> <span class="type">time</span></span><br><span class="line"></span><br><span class="line">browser = webdriver.Chrome()</span><br><span class="line">browser.<span class="keyword">get</span>(<span class="string">'https://www.taobao.com'</span>)</span><br><span class="line">input = browser.find_element_by_id(<span class="string">'q'</span>)</span><br><span class="line"><span class="keyword">input</span>.send_keys(<span class="string">'iPhone'</span>)</span><br><span class="line"><span class="type">time</span>.sleep(<span class="number">1</span>)</span><br><span class="line"><span class="keyword">input</span>.clear()</span><br><span class="line"><span class="keyword">input</span>.send_keys(<span class="string">'iPad'</span>)</span><br><span class="line">button = browser.find_element_by_class_name(<span class="string">'btn-search'</span>)</span><br><span class="line">button.click()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里首先驱动浏览器打开淘宝，然后用<code>find_element_by_id()</code>方法获取输入框，然后用<code>send_keys()</code>方法输入iPhone文字，等待一秒后用<code>clear()</code>方法清空输入框，再次调用<code>send_keys()</code>方法输入iPad文字，之后再用<code>find_element_by_class_name()</code>方法获取搜索按钮，最后调用<code>click()</code>方法完成搜索动作。</p>
                  <p>通过上面的方法，我们就完成了一些常见节点的动作操作，更多的操作可以参见官方文档的交互动作介绍：<a href="http://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.remote.webelement" target="_blank" rel="noopener">http://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.remote.webelement</a>。</p>
                  <h2 id="7-动作链"><a href="#7-动作链" class="headerlink" title="7. 动作链"></a>7. 动作链</h2>
                  <p>在上面的实例中，一些交互动作都是针对某个节点执行的。比如，对于输入框，我们就调用它的输入文字和清空文字方法；对于按钮，就调用它的点击方法。其实，还有另外一些操作，它们没有特定的执行对象，比如鼠标拖曳、键盘按键等，这些动作用另一种方式来执行，那就是动作链。</p>
                  <p>比如，现在实现一个节点的拖曳操作，将某个节点从一处拖曳到另外一处，可以这样实现：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> selenium <span class="keyword">import</span> webdriver</span><br><span class="line"><span class="keyword">from</span> selenium.webdriver <span class="keyword">import</span> ActionChains</span><br><span class="line"></span><br><span class="line">browser = webdriver.Chrome()</span><br><span class="line">url = <span class="string">'http://www.runoob.com/try/try.php?filename=jqueryui-api-droppable'</span></span><br><span class="line">browser.<span class="keyword">get</span>(url)</span><br><span class="line">browser.switch_to.frame(<span class="string">'iframeResult'</span>)</span><br><span class="line">source = browser.find_element_by_css_selector(<span class="string">'#draggable'</span>)</span><br><span class="line">target = browser.find_element_by_css_selector(<span class="string">'#droppable'</span>)</span><br><span class="line">actions = ActionChains(browser)</span><br><span class="line">actions.drag_and_drop(source, target)</span><br><span class="line">actions.<span class="keyword">perform</span>()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>首先，打开网页中的一个拖曳实例，然后依次选中要拖曳的节点和拖曳到的目标节点，接着声明<code>ActionChains</code>对象并将其赋值为<code>actions</code>变量，然后通过调用<code>actions</code>变量的<code>drag_and_drop()</code>方法，再调用<code>perform()</code>方法执行动作，此时就完成了拖曳操作，如图7-4和图7-5所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-4.jpg" alt="">图7-4 拖曳前的页面</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/7-5.jpg" alt="">图7-5 拖曳后的页面</p>
                  <p>更多的动作链操作可以参考官方文档：<a href="http://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.common.action_chains" target="_blank" rel="noopener">http://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.common.action_chains</a>。</p>
                  <h2 id="8-执行JavaScript"><a href="#8-执行JavaScript" class="headerlink" title="8. 执行JavaScript"></a>8. 执行JavaScript</h2>
                  <p>对于某些操作，Selenium API并没有提供。比如，下拉进度条，它可以直接模拟运行JavaScript，此时使用<code>execute_script()</code>方法即可实现，代码如下：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from selenium import webdriver</span><br><span class="line"></span><br><span class="line">browser = webdriver.<span class="constructor">Chrome()</span></span><br><span class="line">browser.get('https:<span class="comment">//www.zhihu.com/explore')</span></span><br><span class="line">browser.execute<span class="constructor">_script('<span class="params">window</span>.<span class="params">scrollTo</span>(0, <span class="params">document</span>.<span class="params">body</span>.<span class="params">scrollHeight</span>)</span>')</span><br><span class="line">browser.execute<span class="constructor">_script('<span class="params">alert</span>(<span class="string">"To Bottom"</span>)</span>')</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里就利用<code>execute_script()</code>方法将进度条下拉到最底部，然后弹出alert提示框。</p>
                  <p>所以说有了这个方法，基本上API没有提供的所有功能都可以用执行JavaScript的方式来实现了。</p>
                  <h2 id="9-获取节点信息"><a href="#9-获取节点信息" class="headerlink" title="9. 获取节点信息"></a>9. 获取节点信息</h2>
                  <p>前面说过，通过<code>page_source</code>属性可以获取网页的源代码，接着就可以使用解析库（如正则表达式、Beautiful Soup、pyquery等）来提取信息了。</p>
                  <p>不过，既然Selenium已经提供了选择节点的方法，返回的是<code>WebElement</code>类型，那么它也有相关的方法和属性来直接提取节点信息，如属性、文本等。这样的话，我们就可以不用通过解析源代码来提取信息了，非常方便。</p>
                  <p>接下来，就看看通过怎样的方式来获取节点信息吧。</p>
                  <h3 id="获取属性"><a href="#获取属性" class="headerlink" title="获取属性"></a>获取属性</h3>
                  <p>我们可以使用<code>get_attribute()</code>方法来获取节点的属性，但是其前提是先选中这个节点，示例如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> selenium import webdriver</span><br><span class="line"><span class="keyword">from</span> selenium.webdriver import ActionChains</span><br><span class="line"></span><br><span class="line">browser = webdriver.Chrome()</span><br><span class="line">url = <span class="string">'https://www.zhihu.com/explore'</span></span><br><span class="line">browser.<span class="builtin-name">get</span>(url)</span><br><span class="line">logo = browser.find_element_by_id(<span class="string">'zh-top-link-logo'</span>)</span><br><span class="line"><span class="builtin-name">print</span>(logo)</span><br><span class="line"><span class="builtin-name">print</span>(logo.get_attribute(<span class="string">'class'</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行之后，程序便会驱动浏览器打开知乎页面，然后获取知乎的logo节点，最后打印出它的<code>class</code>。</p>
                  <p>控制台的输出结果如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;selenium<span class="selector-class">.webdriver</span><span class="selector-class">.remote</span><span class="selector-class">.webelement</span><span class="selector-class">.WebElement</span> (session=<span class="string">"e08c0f28d7f44d75ccd50df6bb676104"</span>, element=<span class="string">"0.7236390660048155-1"</span>)&gt;</span><br><span class="line">zu-<span class="attribute">top</span>-link-logo</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>通过<code>get_attribute()</code>方法，然后传入想要获取的属性名，就可以得到它的值了。</p>
                  <h3 id="获取文本值"><a href="#获取文本值" class="headerlink" title="获取文本值"></a>获取文本值</h3>
                  <p>每个<code>WebElement</code>节点都有<code>text</code>属性，直接调用这个属性就可以得到节点内部的文本信息，这相当于Beautiful Soup的<code>get_text()</code>方法、pyquery的<code>text()</code>方法，示例如下：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> selenium <span class="keyword">import</span> webdriver</span><br><span class="line"></span><br><span class="line">browser = webdriver.Chrome()</span><br><span class="line">url = <span class="string">'https://www.zhihu.com/explore'</span></span><br><span class="line">browser.<span class="keyword">get</span>(url)</span><br><span class="line">input = browser.find_element_by_class_name(<span class="string">'zu-top-add-question'</span>)</span><br><span class="line">print(<span class="keyword">input</span>.text)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里依然先打开知乎页面，然后获取“提问”按钮这个节点，再将其文本值打印出来。</p>
                  <p>控制台的输出结果如下：</p>
                  <figure class="highlight plain">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">提问</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="获取id、位置、标签名和大小"><a href="#获取id、位置、标签名和大小" class="headerlink" title="获取id、位置、标签名和大小"></a>获取id、位置、标签名和大小</h3>
                  <p>另外，<code>WebElement</code>节点还有一些其他属性，比如<code>id</code>属性可以获取节点<code>id</code>，<code>location</code>属性可以获取该节点在页面中的相对位置，<code>tag_name</code>属性可以获取标签名称，<code>size</code>属性可以获取节点的大小，也就是宽高，这些属性有时候还是很有用的。示例如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"> from selenium import webdriver</span><br><span class="line"></span><br><span class="line">browser = webdriver.Chrome()</span><br><span class="line">url = <span class="string">'https://www.zhihu.com/explore'</span></span><br><span class="line">browser.get(url)</span><br><span class="line"><span class="selector-tag">input</span> = browser.find_element_by_class_name(<span class="string">'zu-top-add-question'</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(input.id)</span></span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(input.location)</span></span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(input.tag_name)</span></span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(input.size)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里首先获得“提问”按钮这个节点，然后调用其<code>id</code>、<code>location</code>、<code>tag_name</code>、<code>size</code>属性来获取对应的属性值。</p>
                  <h2 id="10-切换Frame"><a href="#10-切换Frame" class="headerlink" title="10. 切换Frame"></a>10. 切换Frame</h2>
                  <p>我们知道网页中有一种节点叫作iframe，也就是子Frame，相当于页面的子页面，它的结构和外部网页的结构完全一致。Selenium打开页面后，它默认是在父级Frame里面操作，而此时如果页面中还有子Frame，它是不能获取到子Frame里面的节点的。这时就需要使用<code>switch_to.frame()</code>方法来切换Frame。示例如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import time</span><br><span class="line">from selenium import webdriver</span><br><span class="line">from selenium<span class="selector-class">.common</span><span class="selector-class">.exceptions</span> import NoSuchElementException</span><br><span class="line"></span><br><span class="line">browser = webdriver.Chrome()</span><br><span class="line">url = <span class="string">'http://www.runoob.com/try/try.php?filename=jqueryui-api-droppable'</span></span><br><span class="line">browser.get(url)</span><br><span class="line">browser<span class="selector-class">.switch_to</span>.frame(<span class="string">'iframeResult'</span>)</span><br><span class="line">try:</span><br><span class="line">    logo = browser.find_element_by_class_name(<span class="string">'logo'</span>)</span><br><span class="line">except NoSuchElementException:</span><br><span class="line">    print(<span class="string">'NO LOGO'</span>)</span><br><span class="line">browser<span class="selector-class">.switch_to</span>.parent_frame()</span><br><span class="line">logo = browser.find_element_by_class_name(<span class="string">'logo'</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(logo)</span></span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(logo.text)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>控制台的输出如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="literal">NO</span> LOGO</span><br><span class="line">&lt;selenium.webdriver.remote.webelement.WebElement (<span class="attribute">session</span>=<span class="string">"4bb8ac03ced4ecbdefef03ffdc0e4ccd"</span>, <span class="attribute">element</span>=<span class="string">"0.13792611320464965-2"</span>)&gt;</span><br><span class="line">RUNOOB.COM</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里还是以前面演示动作链操作的网页为实例，首先通过<code>switch_to.frame()</code>方法切换到子Frame里面，然后尝试获取父级Frame里的logo节点（这是不能找到的），如果找不到的话，就会抛出<code>NoSuchElementException</code>异常，异常被捕捉之后，就会输出<code>NO LOGO</code>。接下来，重新切换回父级Frame，然后再次重新获取节点，发现此时可以成功获取了。</p>
                  <p>所以，当页面中包含子Frame时，如果想获取子Frame中的节点，需要先调用<code>switch_to.frame()</code>方法切换到对应的Frame，然后再进行操作。</p>
                  <h2 id="11-延时等待"><a href="#11-延时等待" class="headerlink" title="11. 延时等待"></a>11. 延时等待</h2>
                  <p>在Selenium中，<code>get()</code>方法会在网页框架加载结束后结束执行，此时如果获取<code>page_source</code>，可能并不是浏览器完全加载完成的页面，如果某些页面有额外的Ajax请求，我们在网页源代码中也不一定能成功获取到。所以，这里需要延时等待一定时间，确保节点已经加载出来。</p>
                  <p>这里等待的方式有两种：一种是隐式等待，一种是显式等待。</p>
                  <h3 id="隐式等待"><a href="#隐式等待" class="headerlink" title="隐式等待"></a>隐式等待</h3>
                  <p>当使用隐式等待执行测试的时候，如果Selenium没有在DOM中找到节点，将继续等待，超出设定时间后，则抛出找不到节点的异常。换句话说，当查找节点而节点并没有立即出现的时候，隐式等待将等待一段时间再查找DOM，默认的时间是0。示例如下：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from selenium import webdriver</span><br><span class="line"></span><br><span class="line">browser = webdriver.<span class="constructor">Chrome()</span></span><br><span class="line">browser.implicitly<span class="constructor">_wait(10)</span></span><br><span class="line">browser.get('https:<span class="comment">//www.zhihu.com/explore')</span></span><br><span class="line">input = browser.find<span class="constructor">_element_by_class_name('<span class="params">zu</span>-<span class="params">top</span>-<span class="params">add</span>-<span class="params">question</span>')</span></span><br><span class="line">print(input)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们用<code>implicitly_wait()</code>方法实现了隐式等待。</p>
                  <h3 id="显式等待"><a href="#显式等待" class="headerlink" title="显式等待"></a>显式等待</h3>
                  <p>隐式等待的效果其实并没有那么好，因为我们只规定了一个固定时间，而页面的加载时间会受到网络条件的影响。</p>
                  <p>这里还有一种更合适的显式等待方法，它指定要查找的节点，然后指定一个最长等待时间。如果在规定时间内加载出来了这个节点，就返回查找的节点；如果到了规定时间依然没有加载出该节点，则抛出超时异常。示例如下：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> selenium <span class="keyword">import</span> webdriver</span><br><span class="line"><span class="keyword">from</span> selenium.webdriver.common.<span class="keyword">by</span> <span class="keyword">import</span> <span class="keyword">By</span></span><br><span class="line"><span class="keyword">from</span> selenium.webdriver.support.ui <span class="keyword">import</span> WebDriverWait</span><br><span class="line"><span class="keyword">from</span> selenium.webdriver.support <span class="keyword">import</span> expected_conditions <span class="keyword">as</span> EC</span><br><span class="line"></span><br><span class="line">browser = webdriver.Chrome()</span><br><span class="line">browser.<span class="keyword">get</span>(<span class="string">'https://www.taobao.com/'</span>)</span><br><span class="line">wait = WebDriverWait(browser, <span class="number">10</span>)</span><br><span class="line">input = wait.<span class="keyword">until</span>(EC.presence_of_element_located((<span class="keyword">By</span>.ID, <span class="string">'q'</span>)))</span><br><span class="line">button = wait.<span class="keyword">until</span>(EC.element_to_be_clickable((<span class="keyword">By</span>.CSS_SELECTOR, <span class="string">'.btn-search'</span>)))</span><br><span class="line">print(<span class="keyword">input</span>, button)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里首先引入<code>WebDriverWait</code>这个对象，指定最长等待时间，然后调用它的<code>until()</code>方法，传入要等待条件<code>expected_conditions</code>。比如，这里传入了<code>presence_of_element_located</code>这个条件，代表节点出现的意思，其参数是节点的定位元组，也就是<code>ID</code>为<code>q</code>的节点搜索框。</p>
                  <p>这样可以做到的效果就是，在10秒内如果<code>ID</code>为<code>q</code>的节点（即搜索框）成功加载出来，就返回该节点；如果超过10秒还没有加载出来，就抛出异常。</p>
                  <p>对于按钮，可以更改一下等待条件，比如改为<code>element_to_be_clickable</code>，也就是可点击，所以查找按钮时查找CSS选择器为.btn-search的按钮，如果10秒内它是可点击的，也就是成功加载出来了，就返回这个按钮节点；如果超过10秒还不可点击，也就是没有加载出来，就抛出异常。</p>
                  <p>运行代码，在网速较佳的情况下是可以成功加载出来的。</p>
                  <p>控制台的输出如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;selenium.webdriver.remote.webelement.WebElement (<span class="attribute">session</span>=<span class="string">"07dd2fbc2d5b1ce40e82b9754aba8fa8"</span>, <span class="attribute">element</span>=<span class="string">"0.5642646294074107-1"</span>)&gt;</span><br><span class="line">&lt;selenium.webdriver.remote.webelement.WebElement (<span class="attribute">session</span>=<span class="string">"07dd2fbc2d5b1ce40e82b9754aba8fa8"</span>, <span class="attribute">element</span>=<span class="string">"0.5642646294074107-2"</span>)&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，控制台成功输出了两个节点，它们都是<code>WebElement</code>类型。</p>
                  <p>如果网络有问题，10秒内没有成功加载，那就抛出<code>TimeoutException</code>异常，此时控制台的输出如下：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">TimeoutException Traceback (most recent call last)</span><br><span class="line">&lt;ipython-input-<span class="number">4</span>-f3d73973b223&gt; <span class="keyword">in</span> &lt;<span class="keyword">module</span>&gt;<span class="literal">()</span></span><br><span class="line">      <span class="number">7</span> browser.get('https:<span class="comment">//www.taobao.com/')</span></span><br><span class="line">      <span class="number">8</span> wait = <span class="constructor">WebDriverWait(<span class="params">browser</span>, 10)</span></span><br><span class="line">----&gt; <span class="number">9</span> input = wait.until(<span class="module-access"><span class="module"><span class="identifier">EC</span>.</span></span>presence<span class="constructor">_of_element_located((By.ID, '<span class="params">q</span>')</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>关于等待条件，其实还有很多，比如判断标题内容，判断某个节点内是否出现了某文字等。表7-1列出了所有的等待条件。</p>
                  <p>表7-1 等待条件及其含义</p>
                  <p>等待条件</p>
                  <p>含义</p>
                  <p><code>title_is</code></p>
                  <p>标题是某内容</p>
                  <p><code>title_contains</code></p>
                  <p>标题包含某内容</p>
                  <p><code>presence_of_element_located</code></p>
                  <p>节点加载出来，传入定位元组，如<code>(By.ID, &#39;p&#39;)</code></p>
                  <p><code>visibility_of_element_located</code></p>
                  <p>节点可见，传入定位元组</p>
                  <p><code>visibility_of</code></p>
                  <p>可见，传入节点对象</p>
                  <p><code>presence_of_all_elements_located</code></p>
                  <p>所有节点加载出来</p>
                  <p><code>text_to_be_present_in_element</code></p>
                  <p>某个节点文本包含某文字</p>
                  <p><code>text_to_be_present_in_element_value</code></p>
                  <p>某个节点值包含某文字</p>
                  <p><code>frame_to_be_available_and_switch_to_it</code></p>
                  <p>加载并切换</p>
                  <p><code>invisibility_of_element_located</code></p>
                  <p>节点不可见</p>
                  <p><code>element_to_be_clickable</code></p>
                  <p>节点可点击</p>
                  <p><code>staleness_of</code></p>
                  <p>判断一个节点是否仍在DOM，可判断页面是否已经刷新</p>
                  <p><code>element_to_be_selected</code></p>
                  <p>节点可选择，传节点对象</p>
                  <p><code>element_located_to_be_selected</code></p>
                  <p>节点可选择，传入定位元组</p>
                  <p><code>element_selection_state_to_be</code></p>
                  <p>传入节点对象以及状态，相等返回<code>True</code>，否则返回<code>False</code></p>
                  <p><code>element_located_selection_state_to_be</code></p>
                  <p>传入定位元组以及状态，相等返回<code>True</code>，否则返回<code>False</code></p>
                  <p><code>alert_is_present</code></p>
                  <p>是否出现警告</p>
                  <p>关于更多等待条件的参数及用法，可以参考官方文档：<a href="http://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.support.expected_conditions" target="_blank" rel="noopener">http://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.support.expected_conditions</a>。</p>
                  <h2 id="12-前进和后退"><a href="#12-前进和后退" class="headerlink" title="12. 前进和后退"></a>12. 前进和后退</h2>
                  <p>平常使用浏览器时都有前进和后退功能，Selenium也可以完成这个操作，它使用<code>back()</code>方法后退，使用<code>forward()</code>方法前进。示例如下：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> <span class="type">time</span></span><br><span class="line"><span class="keyword">from</span> selenium <span class="keyword">import</span> webdriver</span><br><span class="line"></span><br><span class="line">browser = webdriver.Chrome()</span><br><span class="line">browser.<span class="keyword">get</span>(<span class="string">'https://www.baidu.com/'</span>)</span><br><span class="line">browser.<span class="keyword">get</span>(<span class="string">'https://www.taobao.com/'</span>)</span><br><span class="line">browser.<span class="keyword">get</span>(<span class="string">'https://www.python.org/'</span>)</span><br><span class="line">browser.back()</span><br><span class="line"><span class="type">time</span>.sleep(<span class="number">1</span>)</span><br><span class="line">browser.forward()</span><br><span class="line">browser.<span class="keyword">close</span>()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们连续访问3个页面，然后调用<code>back()</code>方法回到第二个页面，接下来再调用<code>forward()</code>方法又可以前进到第三个页面。</p>
                  <h2 id="13-Cookies"><a href="#13-Cookies" class="headerlink" title="13. Cookies"></a>13. Cookies</h2>
                  <p>使用Selenium，还可以方便地对Cookies进行操作，例如获取、添加、删除Cookies等。示例如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> selenium import webdriver</span><br><span class="line"></span><br><span class="line">browser = webdriver.Chrome()</span><br><span class="line">browser.<span class="builtin-name">get</span>(<span class="string">'https://www.zhihu.com/explore'</span>)</span><br><span class="line"><span class="builtin-name">print</span>(browser.get_cookies())</span><br><span class="line">browser.add_cookie(&#123;<span class="string">'name'</span>: <span class="string">'name'</span>, <span class="string">'domain'</span>: <span class="string">'www.zhihu.com'</span>, <span class="string">'value'</span>: <span class="string">'germey'</span>&#125;)</span><br><span class="line"><span class="builtin-name">print</span>(browser.get_cookies())</span><br><span class="line">browser.delete_all_cookies()</span><br><span class="line"><span class="builtin-name">print</span>(browser.get_cookies())</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>首先，我们访问了知乎。加载完成后，浏览器实际上已经生成Cookies了。接着，调用<code>get_cookies()</code>方法获取所有的Cookies。然后，我们添加一个Cookie，这里传入一个字典，有<code>name</code>、<code>domain</code>和<code>value</code>等内容。接下来，再次获取所有的Cookies。可以发现，结果就多了这一项新加的Cookie。最后，调用<code>delete_all_cookies()</code>方法删除所有的Cookies。再重新获取，发现结果就为空了。</p>
                  <p>控制台的输出如下：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[&#123;<span class="symbol">'secure</span><span class="symbol">':</span> False, <span class="symbol">'value</span><span class="symbol">':</span> '<span class="string">"NGM0ZTM5NDAwMWEyNDQwNDk5ODlkZWY3OTkxY2I0NDY=|1491604091|236e34290a6f407bfbb517888849ea509ac366d0"</span>', <span class="symbol">'domain</span><span class="symbol">':</span> <span class="symbol">'.zhihu.com</span>', <span class="symbol">'path</span><span class="symbol">':</span> <span class="symbol">'/</span>', <span class="symbol">'httpOnly</span><span class="symbol">':</span> False, <span class="symbol">'name</span><span class="symbol">':</span> <span class="symbol">'l_cap_id</span>', <span class="symbol">'expiry</span><span class="symbol">':</span> <span class="number">1494196091.403418</span>&#125;]</span><br><span class="line">[&#123;<span class="symbol">'secure</span><span class="symbol">':</span> False, <span class="symbol">'value</span><span class="symbol">':</span> <span class="symbol">'germey</span>', <span class="symbol">'domain</span><span class="symbol">':</span> <span class="symbol">'.www.zhihu.com</span>', <span class="symbol">'path</span><span class="symbol">':</span> <span class="symbol">'/</span>', <span class="symbol">'httpOnly</span><span class="symbol">':</span> False, <span class="symbol">'name</span><span class="symbol">':</span> <span class="symbol">'name</span>'&#125;, &#123;<span class="symbol">'secure</span><span class="symbol">':</span> False, <span class="symbol">'value</span><span class="symbol">':</span> '<span class="string">"NGM0ZTM5NDAwMWEyNDQwNDk5ODlkZWY3OTkxY2I0NDY=|1491604091|236e34290a6f407bfbb517888849ea509ac366d0"</span>', <span class="symbol">'domain</span><span class="symbol">':</span> <span class="symbol">'.zhihu.com</span>', <span class="symbol">'path</span><span class="symbol">':</span> <span class="symbol">'/</span>', <span class="symbol">'httpOnly</span><span class="symbol">':</span> False, <span class="symbol">'name</span><span class="symbol">':</span> <span class="symbol">'l_cap_id</span>', <span class="symbol">'expiry</span><span class="symbol">':</span> <span class="number">1494196091.403418</span>&#125;]</span><br><span class="line">[]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="14-选项卡管理"><a href="#14-选项卡管理" class="headerlink" title="14. 选项卡管理"></a>14. 选项卡管理</h2>
                  <p>在访问网页的时候，会开启一个个选项卡。在Selenium中，我们也可以对选项卡进行操作。示例如下：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import time</span><br><span class="line">from selenium import webdriver</span><br><span class="line"></span><br><span class="line">browser = webdriver.<span class="constructor">Chrome()</span></span><br><span class="line">browser.get('https:<span class="comment">//www.baidu.com')</span></span><br><span class="line">browser.execute<span class="constructor">_script('<span class="params">window</span>.<span class="params">open</span>()</span>')</span><br><span class="line">print(browser.window_handles)</span><br><span class="line">browser.switch<span class="constructor">_to_window(<span class="params">browser</span>.<span class="params">window_handles</span>[1])</span></span><br><span class="line">browser.get('https:<span class="comment">//www.taobao.com')</span></span><br><span class="line">time.sleep(<span class="number">1</span>)</span><br><span class="line">browser.switch<span class="constructor">_to_window(<span class="params">browser</span>.<span class="params">window_handles</span>[0])</span></span><br><span class="line">browser.get('https:<span class="comment">//python.org')</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>控制台的输出如下：</p>
                  <figure class="highlight lsl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">['CDwindow<span class="number">-4</span>f58e3a7<span class="number">-7167</span><span class="number">-4587</span>-bedf<span class="number">-9</span>cd8c867f435', 'CDwindow<span class="number">-6e05</span>f076<span class="number">-6</span>d77<span class="number">-453</span>a-a36c<span class="number">-32</span>baacc447df']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>首先访问了百度，然后调用了<code>execute_script()</code>方法，这里传入<code>window.open()</code>这个JavaScript语句新开启一个选项卡。接下来，我们想切换到该选项卡。这里调用<code>window_handles</code>属性获取当前开启的所有选项卡，返回的是选项卡的代号列表。要想切换选项卡，只需要调用<code>switch_to_window()</code>方法即可，其中参数是选项卡的代号。这里我们将第二个选项卡代号传入，即跳转到第二个选项卡，接下来在第二个选项卡下打开一个新页面，然后切换回第一个选项卡重新调用switch_to_window()方法，再执行其他操作即可。</p>
                  <h2 id="15-异常处理"><a href="#15-异常处理" class="headerlink" title="15. 异常处理"></a>15. 异常处理</h2>
                  <p>在使用Selenium的过程中，难免会遇到一些异常，例如超时、节点未找到等错误，一旦出现此类错误，程序便不会继续运行了。这里我们可以使用<code>try except</code>语句来捕获各种异常。</p>
                  <p>首先，演示一下节点未找到的异常，示例如下：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> selenium <span class="keyword">import</span> webdriver</span><br><span class="line"></span><br><span class="line">browser = webdriver.Chrome()</span><br><span class="line">browser.<span class="keyword">get</span>(<span class="string">'https://www.baidu.com'</span>)</span><br><span class="line">browser.find_element_by_id(<span class="string">'hello'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里首先打开百度页面，然后尝试选择一个并不存在的节点，此时就会遇到异常。</p>
                  <p>运行之后控制台的输出如下：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">NoSuchElementException Traceback (most recent <span class="keyword">call</span> <span class="keyword">last</span>)</span><br><span class="line"><span class="symbol">&lt;ipython-input-23-978945848a1b&gt;</span> in <span class="symbol">&lt;module&gt;</span>()</span><br><span class="line">      <span class="number">3</span> browser = webdriver.Chrome()</span><br><span class="line">      <span class="number">4</span> browser.<span class="built_in">get</span>(<span class="string">'https://www.baidu.com'</span>)</span><br><span class="line">----&gt; <span class="number">5</span> browser.find_element_by_id(<span class="string">'hello'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，这里抛出了<code>NoSuchElementException</code>异常，这通常是节点未找到的异常。为了防止程序遇到异常而中断，我们需要捕获这些异常，示例如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> selenium <span class="keyword">import</span> webdriver</span><br><span class="line"><span class="keyword">from</span> selenium.common.exceptions <span class="keyword">import</span> TimeoutException, NoSuchElementException</span><br><span class="line"></span><br><span class="line">browser = webdriver.Chrome()</span><br><span class="line"><span class="keyword">try</span>:</span><br><span class="line">    browser.get(<span class="string">'https://www.baidu.com'</span>)</span><br><span class="line"><span class="keyword">except</span> TimeoutException:</span><br><span class="line">    print(<span class="string">'Time Out'</span>)</span><br><span class="line"><span class="keyword">try</span>:</span><br><span class="line">    browser.find_element_by_id(<span class="string">'hello'</span>)</span><br><span class="line"><span class="keyword">except</span> NoSuchElementException:</span><br><span class="line">    print(<span class="string">'No Element'</span>)</span><br><span class="line"><span class="keyword">finally</span>:</span><br><span class="line">    browser.close()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们使用<code>try except</code>来捕获各类异常。比如，我们对<code>find_element_by_id()</code>查找节点的方法捕获<code>NoSuchElementException</code>异常，这样一旦出现这样的错误，就进行异常处理，程序也不会中断了。</p>
                  <p>控制台的输出如下：</p>
                  <figure class="highlight yaml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="literal">No</span> <span class="string">Element</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>关于更多的异常类，可以参考官方文档：<a href="http://selenium-python.readthedocs.io/api.html#module-selenium.common.exceptions" target="_blank" rel="noopener">http://selenium-python.readthedocs.io/api.html#module-selenium.common.exceptions</a>。</p>
                  <p>现在，我们基本对Selenium的常规用法有了大体的了解。使用Selenium，处理JavaScript不再是难事。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-31 20:28:45" itemprop="dateCreated datePublished" datetime="2018-01-31T20:28:45+08:00">2018-01-31</time>
                </span>
                <span id="/5630.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 7.1-Selenium的使用" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>17k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>15 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5627.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5627.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 7-动态渲染页面爬取</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>在前一章中，我们了解了Ajax的分析和抓取方式，这其实也是JavaScript动态渲染的页面的一种情形，通过直接分析Ajax，我们仍然可以借助requests或urllib来实现数据爬取。</p>
                  <p>不过JavaScript动态渲染的页面不止Ajax这一种。比如中国青年网（详见<a href="http://news.youth.cn/gn/" target="_blank" rel="noopener">http://news.youth.cn/gn/</a>），它的分页部分是由JavaScript生成的，并非原始HTML代码，这其中并不包含Ajax请求。比如ECharts的官方实例（详见<a href="http://echarts.baidu.com/demo.html#bar-negative" target="_blank" rel="noopener">http://echarts.baidu.com/demo.html#bar-negative</a>），其图形都是经过JavaScript计算之后生成的。再有淘宝这种页面，它即使是Ajax获取的数据，但是其Ajax接口含有很多加密参数，我们难以直接找出其规律，也很难直接分析Ajax来抓取。</p>
                  <p>为了解决这些问题，我们可以直接使用模拟浏览器运行的方式来实现，这样就可以做到在浏览器中看到是什么样，抓取的源码就是什么样，也就是可见即可爬。这样我们就不用再去管网页内部的JavaScript用了什么算法渲染页面，不用管网页后台的Ajax接口到底有哪些参数。</p>
                  <p>Python提供了许多模拟浏览器运行的库，如Selenium、Splash、PyV8、Ghost等。本章中，我们就来介绍一下Selenium和Splash的用法。有了它们，就不用再为动态渲染的页面发愁了。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-31 20:25:59" itemprop="dateCreated datePublished" datetime="2018-01-31T20:25:59+08:00">2018-01-31</time>
                </span>
                <span id="/5627.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 7-动态渲染页面爬取" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>595</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5616.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5616.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 6.4-分析Ajax爬取今日头条街拍美图</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>本节中，我们以今日头条为例来尝试通过分析Ajax请求来抓取网页数据的方法。这次要抓取的目标是今日头条的街拍美图，抓取完成之后，将每组图片分文件夹下载到本地并保存下来。</p>
                  <h2 id="1-准备工作"><a href="#1-准备工作" class="headerlink" title="1. 准备工作"></a>1. 准备工作</h2>
                  <p>在本节开始之前，请确保已经安装好requests库。如果没有安装，可以参考第1章。</p>
                  <h2 id="2-抓取分析"><a href="#2-抓取分析" class="headerlink" title="2. 抓取分析"></a>2. 抓取分析</h2>
                  <p>在抓取之前，首先要分析抓取的逻辑。打开今日头条的首页<a href="http://www.toutiao.com/" target="_blank" rel="noopener">http://www.toutiao.com/</a>，如图6-15所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/6-15.jpg" alt="">图6-15 首页内容</p>
                  <p>右上角有一个搜索入口，这里尝试抓取街拍美图，所以输入“街拍”二字搜索一下，结果如图6-16所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/6-16.jpg" alt="">图6-16 搜索结果</p>
                  <p>这时打开开发者工具，查看所有的网络请求。首先，打开第一个网络请求，这个请求的URL就是当前的链接<a href="http://epub.ituring.com.cn/article/edit/[http://www.toutiao.com/search/?keyword=%E8%A1%97%E6%8B%8D]" target="_blank" rel="noopener">http://www.toutiao.com/search/?keyword=街拍</a>，打开Preview选项卡查看Response Body。如果页面中的内容是根据第一个请求得到的结果渲染出来的，那么第一个请求的源代码中必然会包含页面结果中的文字。为了验证，我们可以尝试搜索一下搜索结果的标题，比如“路人”二字，如图6-17所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/6-17.jpg" alt="">图6-17 搜索结果</p>
                  <p>我们发现，网页源代码中并没有包含这两个字，搜索匹配结果数目为0。因此，可以初步判断这些内容是由Ajax加载，然后用JavaScript渲染出来的。接下来，我们可以切换到XHR过滤选项卡，查看一下有没有Ajax请求。</p>
                  <p>不出所料，此处出现了一个比较常规的Ajax请求，看看它的结果是否包含了页面中的相关数据。</p>
                  <p>点击<code>data</code>字段展开，发现这里有许多条数据。点击第一条展开，可以发现有一个<code>title</code>字段，它的值正好就是页面中第一条数据的标题。再检查一下其他数据，也正好是一一对应的，如图6-18所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/6-18.jpg" alt="">图6-18 对比结果</p>
                  <p>这就确定了这些数据确实是由Ajax加载的。</p>
                  <p>我们的目的是要抓取其中的美图，这里一组图就对应前面<code>data</code>字段中的一条数据。每条数据还有一个<code>image_detail</code>字段，它是列表形式，这其中就包含了组图的所有图片列表，如图6-19所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/6-19.jpg" alt="">图6-19 图片列表信息</p>
                  <p>因此，我们只需要将列表中的<code>url</code>字段提取出来并下载下来就好了。每一组图都建立一个文件夹，文件夹的名称就为组图的标题。</p>
                  <p>接下来，就可以直接用Python来模拟这个Ajax请求，然后提取出相关美图链接并下载。但是在这之前，我们还需要分析一下URL的规律。</p>
                  <p>切换回Headers选项卡，观察一下它的请求URL和Headers信息，如图6-20所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/6-20.jpg" alt="">图6-20 请求信息</p>
                  <p>可以看到，这是一个GET请求，请求URL的参数有<code>offset</code>、<code>format</code>、<code>keyword</code>、<code>autoload</code>、<code>count</code>和<code>cur_tab</code>。我们需要找出这些参数的规律，因为这样才可以方便地用程序构造出来。</p>
                  <p>接下来，可以滑动页面，多加载一些新结果。在加载的同时可以发现，Network中又出现了许多Ajax请求，如图6-21所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/6-21.jpg" alt="">图6-21 Ajax请求</p>
                  <p>这里观察一下后续链接的参数，发现变化的参数只有<code>offset</code>，其他参数都没有变化，而且第二次请求的<code>offset</code>值为20，第三次为40，第四次为60，所以可以发现规律，这个<code>offset</code>值就是偏移量，进而可以推断出<code>count</code>参数就是一次性获取的数据条数。因此，我们可以用<code>offset</code>参数来控制数据分页。这样一来，我们就可以通过接口批量获取数据了，然后将数据解析，将图片下载下来即可。</p>
                  <h2 id="3-实战演练"><a href="#3-实战演练" class="headerlink" title="3. 实战演练"></a>3. 实战演练</h2>
                  <p>我们刚才已经分析了一下Ajax请求的逻辑，下面就用程序来实现美图下载吧。</p>
                  <p>首先，实现方法<code>get_page()</code>来加载单个Ajax请求的结果。其中唯一变化的参数就是<code>offset</code>，所以我们将它当作参数传递，实现如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">from</span> urllib.parse <span class="keyword">import</span> urlencode</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">get_page</span><span class="params">(offset)</span>:</span></span><br><span class="line">    params = &#123;</span><br><span class="line">        <span class="string">'offset'</span>: offset,</span><br><span class="line">        <span class="string">'format'</span>: <span class="string">'json'</span>,</span><br><span class="line">        <span class="string">'keyword'</span>: <span class="string">'街拍'</span>,</span><br><span class="line">        <span class="string">'autoload'</span>: <span class="string">'true'</span>,</span><br><span class="line">        <span class="string">'count'</span>: <span class="string">'20'</span>,</span><br><span class="line">        <span class="string">'cur_tab'</span>: <span class="string">'1'</span>,</span><br><span class="line">    &#125;</span><br><span class="line">    url = <span class="string">'http://www.toutiao.com/search_content/?'</span> + urlencode(params)</span><br><span class="line">    <span class="keyword">try</span>:</span><br><span class="line">        response = requests.get(url)</span><br><span class="line">        <span class="keyword">if</span> response.status_code == <span class="number">200</span>:</span><br><span class="line">            <span class="keyword">return</span> response.json()</span><br><span class="line">    <span class="keyword">except</span> requests.ConnectionError:</span><br><span class="line">        <span class="keyword">return</span> <span class="literal">None</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们用<code>urlencode()</code>方法构造请求的GET参数，然后用requests请求这个链接，如果返回状态码为200，则调用<code>response</code>的<code>json()</code>方法将结果转为JSON格式，然后返回。</p>
                  <p>接下来，再实现一个解析方法：提取每条数据的<code>image_detail</code>字段中的每一张图片链接，将图片链接和图片所属的标题一并返回，此时可以构造一个生成器。实现代码如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def get_images(json):</span><br><span class="line">    <span class="keyword">if</span> json.<span class="builtin-name">get</span>(<span class="string">'data'</span>):</span><br><span class="line">        <span class="keyword">for</span> item <span class="keyword">in</span> json.<span class="builtin-name">get</span>(<span class="string">'data'</span>):</span><br><span class="line">            title = item.<span class="builtin-name">get</span>(<span class="string">'title'</span>)</span><br><span class="line">            images = item.<span class="builtin-name">get</span>(<span class="string">'image_detail'</span>)</span><br><span class="line">            <span class="keyword">for</span> image <span class="keyword">in</span> images:</span><br><span class="line">                yield &#123;</span><br><span class="line">                    <span class="string">'image'</span>: image.<span class="builtin-name">get</span>(<span class="string">'url'</span>),</span><br><span class="line">                    <span class="string">'title'</span>: title</span><br><span class="line">                &#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来，实现一个保存图片的方法<code>save_image()</code>，其中<code>item</code>就是前面<code>get_images()</code>方法返回的一个字典。在该方法中，首先根据<code>item</code>的<code>title</code>来创建文件夹，然后请求这个图片链接，获取图片的二进制数据，以二进制的形式写入文件。图片的名称可以使用其内容的MD5值，这样可以去除重复。相关代码如下：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import os</span><br><span class="line"><span class="built_in">from</span> hashlib import md5</span><br><span class="line"></span><br><span class="line">def save_image(<span class="keyword">item</span>):</span><br><span class="line">    <span class="keyword">if</span> <span class="keyword">not</span> os.path.exists(<span class="keyword">item</span>.<span class="built_in">get</span>(<span class="string">'title'</span>)):</span><br><span class="line">        os.mkdir(<span class="keyword">item</span>.<span class="built_in">get</span>(<span class="string">'title'</span>))</span><br><span class="line">    <span class="keyword">try</span>:</span><br><span class="line">        response = requests.<span class="built_in">get</span>(<span class="keyword">item</span>.<span class="built_in">get</span>(<span class="string">'image'</span>))</span><br><span class="line">        <span class="keyword">if</span> response.status_code == <span class="number">200</span>:</span><br><span class="line">            file_path = <span class="string">'&#123;0&#125;/&#123;1&#125;.&#123;2&#125;'</span>.<span class="built_in">format</span>(<span class="keyword">item</span>.<span class="built_in">get</span>(<span class="string">'title'</span>), md5(response.content).hexdigest(), <span class="string">'jpg'</span>)</span><br><span class="line">            <span class="keyword">if</span> <span class="keyword">not</span> os.path.exists(file_path):</span><br><span class="line">                <span class="keyword">with</span> <span class="built_in">open</span>(file_path, <span class="string">'wb'</span>) <span class="keyword">as</span> f:</span><br><span class="line">                    f.<span class="built_in">write</span>(response.content)</span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                print(<span class="string">'Already Downloaded'</span>, file_path)</span><br><span class="line">    except requests.ConnectionError:</span><br><span class="line">        print(<span class="string">'Failed to Save Image'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>最后，只需要构造一个<code>offset</code>数组，遍历<code>offset</code>，提取图片链接，并将其下载即可：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> multiprocessing.pool import Pool</span><br><span class="line"></span><br><span class="line">def main(offset):</span><br><span class="line">    json = get_page(offset)</span><br><span class="line">    <span class="keyword">for</span> item <span class="keyword">in</span> get_images(json):</span><br><span class="line">        <span class="builtin-name">print</span>(item)</span><br><span class="line">        save_image(item)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">GROUP_START = 1</span><br><span class="line">GROUP_END = 20</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">'__main__'</span>:</span><br><span class="line">   <span class="built_in"> pool </span>= Pool()</span><br><span class="line">    groups = ([x * 20 <span class="keyword">for</span> x <span class="keyword">in</span> range(GROUP_START, GROUP_END + 1)])</span><br><span class="line">    pool.map(main, groups)</span><br><span class="line">    pool.close()</span><br><span class="line">    pool.join()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里定义了分页的起始页数和终止页数，分别为<code>GROUP_START</code>和<code>GROUP_END</code>，还利用了多线程的线程池，调用其<code>map()</code>方法实现多线程下载。</p>
                  <p>这样整个程序就完成了，运行之后可以发现街拍美图都分文件夹保存下来了，如图6-22所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/6-22.jpg" alt="">图6-22 保存结果</p>
                  <p>最后，我们给出本节的代码地址：<a href="https://github.com/Python3WebSpider/Jiepai" target="_blank" rel="noopener">https://github.com/Python3WebSpider/Jiepai</a>。</p>
                  <p>通过本节，我们了解了Ajax分析的流程、Ajax分页的模拟以及图片的下载过程。</p>
                  <p>本节的内容需要熟练掌握，在后面的实战中我们还会用到很多次这样的分析和抓取。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-30 20:18:22" itemprop="dateCreated datePublished" datetime="2018-01-30T20:18:22+08:00">2018-01-30</time>
                </span>
                <span id="/5616.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 6.4-分析Ajax爬取今日头条街拍美图" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>3.6k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>3 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5609.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5609.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 6.3-Ajax结果提取</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>这里仍然以微博为例，接下来用Python来模拟这些Ajax请求，把我发过的微博爬取下来。</p>
                  <h2 id="1-分析请求"><a href="#1-分析请求" class="headerlink" title="1. 分析请求"></a>1. 分析请求</h2>
                  <p>打开Ajax的XHR过滤器，然后一直滑动页面以加载新的微博内容。可以看到，会不断有Ajax请求发出。</p>
                  <p>选定其中一个请求，分析它的参数信息。点击该请求，进入详情页面，如图6-11所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/6-11.png" alt="">图6-11 详情页面</p>
                  <p>可以发现，这是一个GET类型的请求，请求链接为[<a href="https://m.weibo.cn/api/container/getIndex?type=uid&amp;value=2830678474&amp;containerid=1076032830678474&amp;page=2)。请求的参数有4个：`type`、`value`、`containerid`和`page`。" target="_blank" rel="noopener">https://m.weibo.cn/api/container/getIndex?type=uid&amp;value=2830678474&amp;containerid=1076032830678474&amp;page=2)。请求的参数有4个：`type`、`value`、`containerid`和`page`。</a></p>
                  <p>随后再看看其他请求，可以发现，它们的<code>type</code>、<code>value</code>和<code>containerid</code>始终如一。<code>type</code>始终为<code>uid</code>，<code>value</code>的值就是页面链接中的数字，其实这就是用户的<code>id</code>。另外，还有<code>containerid</code>。可以发现，它就是107603加上用户<code>id</code>。改变的值就是<code>page</code>，很明显这个参数是用来控制分页的，<code>page=1</code>代表第一页，<code>page=2</code>代表第二页，以此类推。</p>
                  <h2 id="2-分析响应"><a href="#2-分析响应" class="headerlink" title="2. 分析响应"></a>2. 分析响应</h2>
                  <p>随后，观察这个请求的响应内容，如图6-12所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/6-12.png" alt="">图6-12 响应内容</p>
                  <p>这个内容是JSON格式的，浏览器开发者工具自动做了解析以方便我们查看。可以看到，最关键的两部分信息就是<code>cardlistInfo</code>和<code>cards</code>：前者包含一个比较重要的信息<code>total</code>，观察后可以发现，它其实是微博的总数量，我们可以根据这个数字来估算分页数；后者则是一个列表，它包含10个元素，展开其中一个看一下，如图6-13所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/6-13.png" alt="">图6-13 列表内容</p>
                  <p>可以发现，这个元素有一个比较重要的字段<code>mblog</code>。展开它，可以发现它包含的正是微博的一些信息，比如<code>attitudes_count</code>（赞数目）、<code>comments_count</code>（评论数目）、<code>reposts_count</code>（转发数目）、<code>created_at</code>（发布时间）、<code>text</code>（微博正文）等，而且它们都是一些格式化的内容。</p>
                  <p>这样我们请求一个接口，就可以得到10条微博，而且请求时只需要改变<code>page</code>参数即可。</p>
                  <p>这样的话，我们只需要简单做一个循环，就可以获取所有微博了。</p>
                  <h2 id="3-实战演练"><a href="#3-实战演练" class="headerlink" title="3. 实战演练"></a>3. 实战演练</h2>
                  <p>这里我们用程序模拟这些Ajax请求，将我的前10页微博全部爬取下来。</p>
                  <p>首先，定义一个方法来获取每次请求的结果。在请求时，<code>page</code>是一个可变参数，所以我们将它作为方法的参数传递进来，相关代码如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> urllib.parse import urlencode</span><br><span class="line">import requests</span><br><span class="line">base_url = <span class="string">'https://m.weibo.cn/api/container/getIndex?'</span></span><br><span class="line"></span><br><span class="line">headers = &#123;</span><br><span class="line">    <span class="string">'Host'</span>: <span class="string">'m.weibo.cn'</span>,</span><br><span class="line">    <span class="string">'Referer'</span>: <span class="string">'https://m.weibo.cn/u/2830678474'</span>,</span><br><span class="line">    <span class="string">'User-Agent'</span>: <span class="string">'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'</span>,</span><br><span class="line">    <span class="string">'X-Requested-With'</span>: <span class="string">'XMLHttpRequest'</span>,</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">def get_page(page):</span><br><span class="line">    params = &#123;</span><br><span class="line">        <span class="string">'type'</span>: <span class="string">'uid'</span>,</span><br><span class="line">        <span class="string">'value'</span>: <span class="string">'2830678474'</span>,</span><br><span class="line">        <span class="string">'containerid'</span>: <span class="string">'1076032830678474'</span>,</span><br><span class="line">        <span class="string">'page'</span>: page</span><br><span class="line">    &#125;</span><br><span class="line">    url = base_url + urlencode(params)</span><br><span class="line">    try:</span><br><span class="line">        response = requests.<span class="builtin-name">get</span>(url, <span class="attribute">headers</span>=headers)</span><br><span class="line">        <span class="keyword">if</span> response.status_code == 200:</span><br><span class="line">            return response.json()</span><br><span class="line">    except requests.ConnectionError as e:</span><br><span class="line">        <span class="builtin-name">print</span>(<span class="string">'Error'</span>, e.args)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>首先，这里定义了<code>base_url</code>来表示请求的URL的前半部分。接下来，构造参数字典，其中<code>type</code>、<code>value</code>和<code>containerid</code>是固定参数，<code>page</code>是可变参数。接下来，调用<code>urlencode()</code>方法将参数转化为URL的GET请求参数，即类似于<code>type=uid&amp;value=2830678474&amp;containerid=1076032830678474&amp;page=2</code>这样的形式。随后，<code>base_url</code>与参数拼合形成一个新的URL。接着，我们用requests请求这个链接，加入<code>headers</code>参数。然后判断响应的状态码，如果是200，则直接调用<code>json()</code>方法将内容解析为JSON返回，否则不返回任何信息。如果出现异常，则捕获并输出其异常信息。</p>
                  <p>随后，我们需要定义一个解析方法，用来从结果中提取想要的信息，比如这次想保存微博的<code>id</code>、正文、赞数、评论数和转发数这几个内容，那么可以先遍历<code>cards</code>，然后获取<code>mblog</code>中的各个信息，赋值为一个新的字典返回即可：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> pyquery import PyQuery as pq</span><br><span class="line"></span><br><span class="line">def parse_page(json):</span><br><span class="line">    <span class="keyword">if</span> json:</span><br><span class="line">        items = json.<span class="builtin-name">get</span>(<span class="string">'data'</span>).<span class="builtin-name">get</span>(<span class="string">'cards'</span>)</span><br><span class="line">        <span class="keyword">for</span> item <span class="keyword">in</span> items:</span><br><span class="line">            item = item.<span class="builtin-name">get</span>(<span class="string">'mblog'</span>)</span><br><span class="line">            weibo = &#123;&#125;</span><br><span class="line">            weibo[<span class="string">'id'</span>] = item.<span class="builtin-name">get</span>(<span class="string">'id'</span>)</span><br><span class="line">            weibo[<span class="string">'text'</span>] = pq(item.<span class="builtin-name">get</span>(<span class="string">'text'</span>)).text()</span><br><span class="line">            weibo[<span class="string">'attitudes'</span>] = item.<span class="builtin-name">get</span>(<span class="string">'attitudes_count'</span>)</span><br><span class="line">            weibo[<span class="string">'comments'</span>] = item.<span class="builtin-name">get</span>(<span class="string">'comments_count'</span>)</span><br><span class="line">            weibo[<span class="string">'reposts'</span>] = item.<span class="builtin-name">get</span>(<span class="string">'reposts_count'</span>)</span><br><span class="line">            yield weibo</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们借助pyquery将正文中的HTML标签去掉。</p>
                  <p>最后，遍历一下<code>page</code>，一共10页，将提取到的结果打印输出即可：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">if</span> __name__ == <span class="string">'__main__'</span>:</span><br><span class="line">    <span class="keyword">for</span><span class="built_in"> page </span><span class="keyword">in</span> range(1, 11):</span><br><span class="line">        json = get_page(page)</span><br><span class="line">        results = parse_page(json)</span><br><span class="line">        <span class="keyword">for</span> result <span class="keyword">in</span> results:</span><br><span class="line">            <span class="builtin-name">print</span>(result)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外，我们还可以加一个方法将结果保存到MongoDB数据库：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> pymongo import MongoClient</span><br><span class="line"></span><br><span class="line">client = MongoClient()</span><br><span class="line">db = client[<span class="string">'weibo'</span>]</span><br><span class="line">collection = db[<span class="string">'weibo'</span>]</span><br><span class="line"></span><br><span class="line">def save_to_mongo(result):</span><br><span class="line">    <span class="keyword">if</span> collection.insert(result):</span><br><span class="line">        <span class="builtin-name">print</span>(<span class="string">'Saved to Mongo'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样所有功能就实现完成了。运行程序后，样例输出结果如下：</p>
                  <figure class="highlight puppet">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;<span class="string">'id'</span>: <span class="string">'4134879836735238'</span>, <span class="string">'text'</span>: <span class="string">'惊不惊喜，刺不刺激，意不意外，感不感动'</span>, <span class="string">'attitudes'</span>: 3, <span class="string">'comments'</span>: 1, <span class="string">'reposts'</span>: 0&#125;</span><br><span class="line">Saved to Mongo</span><br><span class="line">&#123;<span class="string">'id'</span>: <span class="string">'4143853554221385'</span>, <span class="string">'text'</span>: <span class="string">'曾经梦想仗剑走天涯，后来过安检给收走了。分享单曲 远走高飞'</span>, <span class="string">'attitudes'</span>: 5, <span class="string">'comments'</span>: 1, <span class="string">'reposts'</span>: 0&#125;</span><br><span class="line">Saved to Mongo</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>查看一下MongoDB，相应的数据也被保存到MongoDB，如图6-14所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/6-14.png" alt="">图6-14 保存结果</p>
                  <p>这样，我们就顺利通过分析Ajax并编写爬虫爬取下来了微博列表，最后，给出本节的代码地址：<a href="https://github.com/Python3WebSpider/WeiboList" target="_blank" rel="noopener">https://github.com/Python3WebSpider/WeiboList</a>。</p>
                  <p>本节的目的是为了演示Ajax的模拟请求过程，爬取的结果不是重点。该程序仍有很多可以完善的地方，如页码的动态计算、微博查看全文等，若感兴趣，可以尝试一下。</p>
                  <p>通过这个实例，我们主要学会了怎样去分析Ajax请求，怎样用程序来模拟抓取Ajax请求。了解了抓取原理之后，下一节的Ajax实战演练会更加得心应手。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-30 20:06:07" itemprop="dateCreated datePublished" datetime="2018-01-30T20:06:07+08:00">2018-01-30</time>
                </span>
                <span id="/5609.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 6.3-Ajax结果提取" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>3.5k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>3 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5597.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5597.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 6.2-Ajax分析方法</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>这里还以前面的微博为例，我们知道拖动刷新的内容由Ajax加载，而且页面的URL没有变化，那么应该到哪里去查看这些Ajax请求呢？</p>
                  <h2 id="1-查看请求"><a href="#1-查看请求" class="headerlink" title="1. 查看请求"></a>1. 查看请求</h2>
                  <p>这里还需要借助浏览器的开发者工具，下面以Chrome浏览器为例来介绍。</p>
                  <p>首先，用Chrome浏览器打开微博的链接<a href="https://m.weibo.cn/u/2830678474" target="_blank" rel="noopener">https://m.weibo.cn/u/2830678474</a>，随后在页面中点击鼠标右键，从弹出的快捷菜单中选择“检查”选项，此时便会弹出开发者工具，如图6-2所示：</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/6-2.png" alt="">图6-2 开发者工具</p>
                  <p>此时在Elements选项卡中便会观察到网页的源代码，右侧便是节点的样式。</p>
                  <p>不过这不是我们想要寻找的内容。切换到Network选项卡，随后重新刷新页面，可以发现这里出现了非常多的条目，如图6-3所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/6-3.png" alt="">图6-3 Network面板结果</p>
                  <p>前面也提到过，这里其实就是在页面加载过程中浏览器与服务器之间发送请求和接收响应的所有记录。</p>
                  <p>Ajax其实有其特殊的请求类型，它叫作<code>xhr</code>。在图6-3中，我们可以发现一个名称以getIndex开头的请求，其Type为<code>xhr</code>，这就是一个Ajax请求。用鼠标点击这个请求，可以查看这个请求的详细信息，如图6-4所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/6-4.png" alt="">图6-4 详细信息</p>
                  <p>在右侧可以观察到其Request Headers、URL和Response Headers等信息。其中Request Headers中有一个信息为X-Requested-With:XMLHttpRequest，这就标记了此请求是Ajax请求，如图6-5所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/6-5.png" alt="">图6-5 详细信息</p>
                  <p>随后点击一下Preview，即可看到响应的内容，它是JSON格式的。这里Chrome为我们自动做了解析，点击箭头即可展开和收起相应内容，如图6-6所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/6-6.png" alt="">图6-6 JSON结果</p>
                  <p>观察可以发现，这里的返回结果是我的个人信息，如昵称、简介、头像等，这也是用来渲染个人主页所使用的数据。JavaScript接收到这些数据之后，再执行相应的渲染方法，整个页面就渲染出来了。</p>
                  <p>另外，也可以切换到Response选项卡，从中观察到真实的返回数据，如图6-7所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/6-7.png" alt="">图6-7 Response内容</p>
                  <p>接下来，切回到第一个请求，观察一下它的Response是什么，如图6-8所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/6-8.png" alt="">图6-8 Response内容</p>
                  <p>这是最原始的链接<a href="https://m.weibo.cn/u/2830678474" target="_blank" rel="noopener">https://m.weibo.cn/u/2830678474</a>返回的结果，其代码只有不到50行，结构也非常简单，只是执行了一些JavaScript。</p>
                  <p>所以说，我们看到的微博页面的真实数据并不是最原始的页面返回的，而是后来执行JavaScript后再次向后台发送了Ajax请求，浏览器拿到数据后再进一步渲染出来的。</p>
                  <h2 id="2-过滤请求"><a href="#2-过滤请求" class="headerlink" title="2. 过滤请求"></a>2. 过滤请求</h2>
                  <p>接下来，再利用Chrome开发者工具的筛选功能筛选出所有的Ajax请求。在请求的上方有一层筛选栏，直接点击XHR，此时在下方显示的所有请求便都是Ajax请求了，如图6-9所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/6-9.png" alt="">图6-9 Ajax请求</p>
                  <p>接下来，不断滑动页面，可以看到页面底部有一条条新的微博被刷出，而开发者工具下方也一个个地出现Ajax请求，这样我们就可以捕获到所有的Ajax请求了。</p>
                  <p>随意点开一个条目，都可以清楚地看到其Request URL、Request Headers、Response Headers、Response Body等内容，此时想要模拟请求和提取就非常简单了。</p>
                  <p>图6-10所示的内容便是我的某一页微博的列表信息。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/6-10.png" alt="">图6-10 微博列表信息</p>
                  <p>到现在为止，我们已经可以分析出来Ajax请求的一些详细信息了，接下来只需要用程序模拟这些Ajax请求，就可以轻松提取我们所需要的信息了。</p>
                  <p>在下一节中，我们用Python实现Ajax请求的模拟，从而实现数据的抓取。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-30 19:59:19" itemprop="dateCreated datePublished" datetime="2018-01-30T19:59:19+08:00">2018-01-30</time>
                </span>
                <span id="/5597.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 6.2-Ajax分析方法" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>1.5k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5593.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5593.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 6.1-什么是Ajax</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>Ajax，全称为Asynchronous JavaScript and XML，即异步的JavaScript和XML。它不是一门编程语言，而是利用JavaScript在保证页面不被刷新、页面链接不改变的情况下与服务器交换数据并更新部分网页的技术。</p>
                  <p>对于传统的网页，如果想更新其内容，那么必须要刷新整个页面，但有了Ajax，便可以在页面不被全部刷新的情况下更新其内容。在这个过程中，页面实际上是在后台与服务器进行了数据交互，获取到数据之后，再利用JavaScript改变网页，这样网页内容就会更新了。</p>
                  <p>可以到W3School上体验几个示例来感受一下：<a href="http://www.w3school.com.cn/ajax/ajax_xmlhttprequest_send.asp" target="_blank" rel="noopener">http://www.w3school.com.cn/ajax/ajax_xmlhttprequest_send.asp</a>。</p>
                  <h2 id="1-实例引入"><a href="#1-实例引入" class="headerlink" title="1. 实例引入"></a>1. 实例引入</h2>
                  <p>浏览网页的时候，我们会发现很多网页都有下滑查看更多的选项。比如，拿微博来说，我们以我的个人的主页为例：<a href="https://m.weibo.cn/u/2830678474" target="_blank" rel="noopener">https://m.weibo.cn/u/2830678474</a>，切换到微博页面，一直下滑，可以发现下滑几个微博之后，再向下就没有了，转而会出现一个加载的动画，不一会儿下方就继续出现了新的微博内容，这个过程其实就是Ajax加载的过程，如图6-1所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/6-1.png" alt="">图6-1 页面加载过程</p>
                  <p>我们注意到页面其实并没有整个刷新，也就意味着页面的链接没有变化，但是网页中却多了新内容，也就是后面刷出来的新微博。这就是通过Ajax获取新数据并呈现的过程。</p>
                  <h2 id="2-基本原理"><a href="#2-基本原理" class="headerlink" title="2. 基本原理"></a>2. 基本原理</h2>
                  <p>初步了解了Ajax之后，我们再来详细了解它的基本原理。发送Ajax请求到网页更新的这个过程可以简单分为以下3步：</p>
                  <p>(1) 发送请求； (2) 解析内容； (3) 渲染网页。</p>
                  <p>下面我们分别来详细介绍这几个过程。</p>
                  <h3 id="发送请求"><a href="#发送请求" class="headerlink" title="发送请求"></a>发送请求</h3>
                  <p>我们知道JavaScript可以实现页面的各种交互功能，Ajax也不例外，它也是由JavaScript实现的，实际上执行了如下代码：</p>
                  <figure class="highlight javascript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">var</span> xmlhttp;</span><br><span class="line"><span class="keyword">if</span> (<span class="built_in">window</span>.XMLHttpRequest) &#123;</span><br><span class="line">    <span class="comment">// code for IE7+, Firefox, Chrome, Opera, Safari</span></span><br><span class="line">    xmlhttp=<span class="keyword">new</span> XMLHttpRequest();</span><br><span class="line">&#125; <span class="keyword">else</span> &#123;<span class="comment">// code for IE6, IE5</span></span><br><span class="line">    xmlhttp=<span class="keyword">new</span> ActiveXObject(<span class="string">"Microsoft.XMLHTTP"</span>);</span><br><span class="line">&#125;</span><br><span class="line">xmlhttp.onreadystatechange=<span class="function"><span class="keyword">function</span>(<span class="params"></span>) </span>&#123;</span><br><span class="line">    <span class="keyword">if</span> (xmlhttp.readyState==<span class="number">4</span> &amp;&amp; xmlhttp.status==<span class="number">200</span>) &#123;</span><br><span class="line">        <span class="built_in">document</span>.getElementById(<span class="string">"myDiv"</span>).innerHTML=xmlhttp.responseText;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line">xmlhttp.open(<span class="string">"POST"</span>,<span class="string">"/ajax/"</span>,<span class="literal">true</span>);</span><br><span class="line">xmlhttp.send();</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这是JavaScript对Ajax最底层的实现，实际上就是新建了<code>XMLHttpRequest</code>对象，然后调用<code>onreadystatechange</code>属性设置了监听，然后调用<code>open()</code>和<code>send()</code>方法向某个链接（也就是服务器）发送了请求。前面用Python实现请求发送之后，可以得到响应结果，但这里请求的发送变成JavaScript来完成.由于设置了监听，所以当服务器返回响应时，<code>onreadystatechange</code>对应的方法便会被触发，然后在这个方法里面解析响应内容即可。</p>
                  <h3 id="解析内容"><a href="#解析内容" class="headerlink" title="解析内容"></a>解析内容</h3>
                  <p>得到响应之后，<code>onreadystatechange</code>属性对应的方法便会被触发，此时利用<code>xmlhttp</code>的<code>responseText</code>属性便可取到响应内容。这类似于Python中利用requests向服务器发起请求，然后得到响应的过程。那么返回内容可能是HTML，可能是JSON，接下来只需要在方法中用JavaScript进一步处理即可。比如，如果是JSON的话，可以进行解析和转化。</p>
                  <h3 id="渲染网页"><a href="#渲染网页" class="headerlink" title="渲染网页"></a>渲染网页</h3>
                  <p>JavaScript有改变网页内容的能力，解析完响应内容之后，就可以调用JavaScript来针对解析完的内容对网页进行下一步处理了。比如，通过<code>document.getElementById().innerHTML</code>这样的操作，便可以对某个元素内的源代码进行更改，这样网页显示的内容就改变了，这样的操作也被称作DOM操作，即对Document网页文档进行操作，如更改、删除等。</p>
                  <p>上例中，<code>document.getElementById(&quot;myDiv&quot;).innerHTML=xmlhttp.responseText</code>便将<code>ID</code>为<code>myDiv</code>的节点内部的HTML代码更改为服务器返回的内容，这样<code>myDiv</code>元素内部便会呈现出服务器返回的新数据，网页的部分内容看上去就更新了。</p>
                  <p>我们观察到，这3个步骤其实都是由JavaScript完成的，它完成了整个请求、解析和渲染的过程。</p>
                  <p>再回想微博的下拉刷新，这其实就是JavaScript向服务器发送了一个Ajax请求，然后获取新的微博数据，将其解析，并将其渲染在网页中。</p>
                  <p>因此，我们知道，真实的数据其实都是一次次Ajax请求得到的，如果想要抓取这些数据，需要知道这些请求到底是怎么发送的，发往哪里，发了哪些参数。如果我们知道了这些，不就可以用Python模拟这个发送操作，获取到其中的结果了吗？</p>
                  <p>在下一节中，我们就来了解下到哪里可以看到这些后台Ajax操作，去了解它到底是怎么发送的，发送了什么参数。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-30 19:55:17" itemprop="dateCreated datePublished" datetime="2018-01-30T19:55:17+08:00">2018-01-30</time>
                </span>
                <span id="/5593.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 6.1-什么是Ajax" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>2.3k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>2 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5590.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5590.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 6-Ajax数据爬取</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>有时候我们在用requests抓取页面的时候，得到的结果可能和在浏览器中看到的不一样：在浏览器中可以看到正常显示的页面数据，但是使用requests得到的结果并没有。这是因为requests获取的都是原始的HTML文档，而浏览器中的页面则是经过JavaScript处理数据后生成的结果，这些数据的来源有多种，可能是通过Ajax加载的，可能是包含在HTML文档中的，也可能是经过JavaScript和特定算法计算后生成的。</p>
                  <p>对于第一种情况，数据加载是一种异步加载方式，原始的页面最初不会包含某些数据，原始页面加载完后，会再向服务器请求某个接口获取数据，然后数据才被处理从而呈现到网页上，这其实就是发送了一个Ajax请求。</p>
                  <p>照Web发展的趋势来看，这种形式的页面越来越多。网页的原始HTML文档不会包含任何数据，数据都是通过Ajax统一加载后再呈现出来的，这样在Web开发上可以做到前后端分离，而且降低服务器直接渲染页面带来的压力。</p>
                  <p>所以如果遇到这样的页面，直接利用requests等库来抓取原始页面，是无法获取到有效数据的，这时需要分析网页后台向接口发送的Ajax请求，如果可以用requests来模拟Ajax请求，那么就可以成功抓取了。</p>
                  <p>所以，本章我们的主要目的是了解什么是Ajax以及如何去分析和抓取Ajax请求。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-30 19:51:31" itemprop="dateCreated datePublished" datetime="2018-01-30T19:51:31+08:00">2018-01-30</time>
                </span>
                <span id="/5590.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 6-Ajax数据爬取" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>554</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <script>
              document.querySelectorAll('.random').forEach(item => item.src = "https://picsum.photos/id/" + Math.floor(Math.random() * Math.floor(300)) + "/200/133")

            </script>
            <nav class="pagination">
              <a class="extend prev" rel="prev" href="/page/15/"><i class="fa fa-angle-left" aria-label="上一页"></i></a><a class="page-number" href="/">1</a><span class="space">&hellip;</span><a class="page-number" href="/page/15/">15</a><span class="page-number current">16</span><a class="page-number" href="/page/17/">17</a><span class="space">&hellip;</span><a class="page-number" href="/page/31/">31</a><a class="extend next" rel="next" href="/page/17/"><i class="fa fa-angle-right" aria-label="下一页"></i></a>
            </nav>
          </div>
          <script>
            window.addEventListener('tabs:register', () =>
            {
              let
              {
                activeClass
              } = CONFIG.comments;
              if (CONFIG.comments.storage)
              {
                activeClass = localStorage.getItem('comments_active') || activeClass;
              }
              if (activeClass)
              {
                let activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
                if (activeTab)
                {
                  activeTab.click();
                }
              }
            });
            if (CONFIG.comments.storage)
            {
              window.addEventListener('tabs:click', event =>
              {
                if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
                let commentClass = event.target.classList[1];
                localStorage.setItem('comments_active', commentClass);
              });
            }

          </script>
        </div>
        <div class="toggle sidebar-toggle">
          <span class="toggle-line toggle-line-first"></span>
          <span class="toggle-line toggle-line-middle"></span>
          <span class="toggle-line toggle-line-last"></span>
        </div>
        <aside class="sidebar">
          <div class="sidebar-inner">
            <ul class="sidebar-nav motion-element">
              <li class="sidebar-nav-toc"> 文章目录 </li>
              <li class="sidebar-nav-overview"> 站点概览 </li>
            </ul>
            <!--noindex-->
            <div class="post-toc-wrap sidebar-panel">
            </div>
            <!--/noindex-->
            <div class="site-overview-wrap sidebar-panel">
              <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
                <img class="site-author-image" itemprop="image" alt="崔庆才" src="/images/avatar.png">
                <p class="site-author-name" itemprop="name">崔庆才</p>
                <div class="site-description" itemprop="description">崔庆才的个人站点，记录生活的瞬间，分享学习的心得。</div>
              </div>
              <div class="site-state-wrap motion-element">
                <nav class="site-state">
                  <div class="site-state-item site-state-posts">
                    <a href="/archives/">
                      <span class="site-state-item-count">608</span>
                      <span class="site-state-item-name">日志</span>
                    </a>
                  </div>
                  <div class="site-state-item site-state-categories">
                    <a href="/categories/">
                      <span class="site-state-item-count">24</span>
                      <span class="site-state-item-name">分类</span></a>
                  </div>
                  <div class="site-state-item site-state-tags">
                    <a href="/tags/">
                      <span class="site-state-item-count">156</span>
                      <span class="site-state-item-name">标签</span></a>
                  </div>
                </nav>
              </div>
              <div class="links-of-author motion-element">
                <span class="links-of-author-item">
                  <a href="https://github.com/Germey" title="GitHub → https:&#x2F;&#x2F;github.com&#x2F;Germey" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>GitHub</a>
                </span>
                <span class="links-of-author-item">
                  <a href="mailto:cqc@cuiqingcai.com.com" title="邮件 → mailto:cqc@cuiqingcai.com.com" rel="noopener" target="_blank"><i class="fa fa-envelope fa-fw"></i>邮件</a>
                </span>
                <span class="links-of-author-item">
                  <a href="https://weibo.com/cuiqingcai" title="微博 → https:&#x2F;&#x2F;weibo.com&#x2F;cuiqingcai" rel="noopener" target="_blank"><i class="fab fa-weibo fa-fw"></i>微博</a>
                </span>
                <span class="links-of-author-item">
                  <a href="https://www.zhihu.com/people/Germey" title="知乎 → https:&#x2F;&#x2F;www.zhihu.com&#x2F;people&#x2F;Germey" rel="noopener" target="_blank"><i class="fa fa-magic fa-fw"></i>知乎</a>
                </span>
              </div>
            </div>
            <div style=" width: 100%;" class="sidebar-panel sidebar-panel-image sidebar-panel-active">
              <a href="https://tutorial.lengyue.video/?coupon=12ef4b1a-a3db-11ea-bb37-0242ac130002_cqx_850" target="_blank" rel="noopener">
                <img src="https://qiniu.cuiqingcai.com/bco2a.png" style=" width: 100%;">
              </a>
            </div>
            <div style=" width: 100%;" class="sidebar-panel sidebar-panel-image sidebar-panel-active">
              <a href="http://www.ipidea.net/?utm-source=cqc&utm-keyword=?cqc" target="_blank" rel="noopener">
                <img src="https://qiniu.cuiqingcai.com/0ywun.png" style=" width: 100%;">
              </a>
            </div>
            <div class="sidebar-panel sidebar-panel-tags sidebar-panel-active">
              <h4 class="name"> 标签云 </h4>
              <div class="content">
                <a href="/tags/2048/" style="font-size: 10px;">2048</a> <a href="/tags/API/" style="font-size: 10px;">API</a> <a href="/tags/Bootstrap/" style="font-size: 11.25px;">Bootstrap</a> <a href="/tags/CDN/" style="font-size: 10px;">CDN</a> <a href="/tags/CQC/" style="font-size: 10px;">CQC</a> <a href="/tags/CSS/" style="font-size: 10px;">CSS</a> <a href="/tags/CSS-%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 10px;">CSS 反爬虫</a> <a href="/tags/CV/" style="font-size: 10px;">CV</a> <a href="/tags/Django/" style="font-size: 10px;">Django</a> <a href="/tags/Eclipse/" style="font-size: 11.25px;">Eclipse</a> <a href="/tags/FTP/" style="font-size: 10px;">FTP</a> <a href="/tags/Git/" style="font-size: 10px;">Git</a> <a href="/tags/GitHub/" style="font-size: 13.75px;">GitHub</a> <a href="/tags/HTML5/" style="font-size: 10px;">HTML5</a> <a href="/tags/Hexo/" style="font-size: 10px;">Hexo</a> <a href="/tags/IT/" style="font-size: 10px;">IT</a> <a href="/tags/JSP/" style="font-size: 10px;">JSP</a> <a href="/tags/JavaScript/" style="font-size: 10px;">JavaScript</a> <a href="/tags/K8s/" style="font-size: 10px;">K8s</a> <a href="/tags/LOGO/" style="font-size: 10px;">LOGO</a> <a href="/tags/Linux/" style="font-size: 10px;">Linux</a> <a href="/tags/MIUI/" style="font-size: 10px;">MIUI</a> <a href="/tags/MongoDB/" style="font-size: 10px;">MongoDB</a> <a href="/tags/Mysql/" style="font-size: 10px;">Mysql</a> <a href="/tags/NBA/" style="font-size: 10px;">NBA</a> <a href="/tags/PHP/" style="font-size: 11.25px;">PHP</a> <a href="/tags/PS/" style="font-size: 10px;">PS</a> <a href="/tags/Pathlib/" style="font-size: 10px;">Pathlib</a> <a href="/tags/PhantomJS/" style="font-size: 10px;">PhantomJS</a> <a href="/tags/Python/" style="font-size: 15px;">Python</a> <a href="/tags/Python3/" style="font-size: 12.5px;">Python3</a> <a href="/tags/Pythonic/" style="font-size: 10px;">Pythonic</a> <a href="/tags/QQ/" style="font-size: 10px;">QQ</a> <a href="/tags/Redis/" style="font-size: 10px;">Redis</a> <a href="/tags/SAE/" style="font-size: 10px;">SAE</a> <a href="/tags/SSH/" style="font-size: 10px;">SSH</a> <a href="/tags/SVG/" style="font-size: 10px;">SVG</a> <a href="/tags/Scrapy/" style="font-size: 10px;">Scrapy</a> <a href="/tags/Scrapy-redis/" style="font-size: 10px;">Scrapy-redis</a> <a href="/tags/Scrapy%E5%88%86%E5%B8%83%E5%BC%8F/" style="font-size: 10px;">Scrapy分布式</a> <a href="/tags/Selenium/" style="font-size: 10px;">Selenium</a> <a href="/tags/TKE/" style="font-size: 10px;">TKE</a> <a href="/tags/Ubuntu/" style="font-size: 11.25px;">Ubuntu</a> <a href="/tags/VS-Code/" style="font-size: 10px;">VS Code</a> <a href="/tags/Vs-Code/" style="font-size: 10px;">Vs Code</a> <a href="/tags/Vue/" style="font-size: 11.25px;">Vue</a> <a href="/tags/Webpack/" style="font-size: 10px;">Webpack</a> <a href="/tags/Windows/" style="font-size: 10px;">Windows</a> <a href="/tags/Winpcap/" style="font-size: 10px;">Winpcap</a> <a href="/tags/WordPress/" style="font-size: 13.75px;">WordPress</a> <a href="/tags/Youtube/" style="font-size: 11.25px;">Youtube</a> <a href="/tags/android/" style="font-size: 10px;">android</a> <a href="/tags/ansible/" style="font-size: 10px;">ansible</a> <a href="/tags/cocos2d-x/" style="font-size: 10px;">cocos2d-x</a> <a href="/tags/e6/" style="font-size: 10px;">e6</a> <a href="/tags/fitvids/" style="font-size: 10px;">fitvids</a> <a href="/tags/git/" style="font-size: 11.25px;">git</a> <a href="/tags/json/" style="font-size: 10px;">json</a> <a href="/tags/js%E9%80%86%E5%90%91/" style="font-size: 10px;">js逆向</a> <a href="/tags/kubernetes/" style="font-size: 10px;">kubernetes</a> <a href="/tags/log/" style="font-size: 10px;">log</a> <a href="/tags/logging/" style="font-size: 10px;">logging</a> <a href="/tags/matlab/" style="font-size: 11.25px;">matlab</a> <a href="/tags/python/" style="font-size: 20px;">python</a> <a href="/tags/pytube/" style="font-size: 11.25px;">pytube</a> <a href="/tags/pywin32/" style="font-size: 10px;">pywin32</a> <a href="/tags/style/" style="font-size: 10px;">style</a> <a href="/tags/tomcat/" style="font-size: 10px;">tomcat</a> <a href="/tags/ubuntu/" style="font-size: 10px;">ubuntu</a> <a href="/tags/uwsgi/" style="font-size: 10px;">uwsgi</a> <a href="/tags/vsftpd/" style="font-size: 10px;">vsftpd</a> <a href="/tags/wamp/" style="font-size: 10px;">wamp</a> <a href="/tags/wineQQ/" style="font-size: 10px;">wineQQ</a> <a href="/tags/%E4%B8%83%E7%89%9B/" style="font-size: 11.25px;">七牛</a> <a href="/tags/%E4%B8%8A%E6%B5%B7/" style="font-size: 10px;">上海</a> <a href="/tags/%E4%B8%AA%E4%BA%BA%E7%BD%91%E7%AB%99/" style="font-size: 10px;">个人网站</a> <a href="/tags/%E4%B8%BB%E9%A2%98/" style="font-size: 10px;">主题</a> <a href="/tags/%E4%BA%91%E4%BA%A7%E5%93%81/" style="font-size: 10px;">云产品</a> <a href="/tags/%E4%BA%91%E5%AD%98%E5%82%A8/" style="font-size: 10px;">云存储</a> <a href="/tags/%E4%BA%AC%E4%B8%9C%E4%BA%91/" style="font-size: 10px;">京东云</a> <a href="/tags/%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD/" style="font-size: 12.5px;">人工智能</a> <a href="/tags/%E4%BB%A3%E7%90%86/" style="font-size: 10px;">代理</a> <a href="/tags/%E4%BB%A3%E7%A0%81/" style="font-size: 10px;">代码</a> <a href="/tags/%E4%BB%A3%E7%A0%81%E5%88%86%E4%BA%AB%E5%9B%BE/" style="font-size: 10px;">代码分享图</a> <a href="/tags/%E4%BC%98%E5%8C%96/" style="font-size: 10px;">优化</a> <a href="/tags/%E4%BD%8D%E8%BF%90%E7%AE%97/" style="font-size: 10px;">位运算</a> <a href="/tags/%E5%85%AC%E4%BC%97%E5%8F%B7/" style="font-size: 10px;">公众号</a> <a href="/tags/%E5%88%86%E4%BA%AB/" style="font-size: 10px;">分享</a> <a href="/tags/%E5%88%86%E5%B8%83%E5%BC%8F/" style="font-size: 10px;">分布式</a> <a href="/tags/%E5%88%9B%E4%B8%9A/" style="font-size: 10px;">创业</a> <a href="/tags/%E5%89%8D%E7%AB%AF/" style="font-size: 12.5px;">前端</a> <a href="/tags/%E5%8D%9A%E5%AE%A2/" style="font-size: 10px;">博客</a> <a href="/tags/%E5%8E%9F%E7%94%9FAPP/" style="font-size: 10px;">原生APP</a> <a href="/tags/%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 12.5px;">反爬虫</a> <a href="/tags/%E5%91%BD%E4%BB%A4/" style="font-size: 10px;">命令</a> <a href="/tags/%E5%93%8D%E5%BA%94%E5%BC%8F%E5%B8%83%E5%B1%80/" style="font-size: 10px;">响应式布局</a> <a href="/tags/%E5%9E%83%E5%9C%BE%E9%82%AE%E4%BB%B6/" style="font-size: 10px;">垃圾邮件</a> <a href="/tags/%E5%9F%9F%E5%90%8D%E7%BB%91%E5%AE%9A/" style="font-size: 10px;">域名绑定</a> <a href="/tags/%E5%A4%8D%E7%9B%98/" style="font-size: 10px;">复盘</a> <a href="/tags/%E5%A4%A7%E4%BC%97%E7%82%B9%E8%AF%84/" style="font-size: 10px;">大众点评</a> <a href="/tags/%E5%AD%97%E4%BD%93%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 10px;">字体反爬虫</a> <a href="/tags/%E5%AD%97%E7%AC%A6%E9%97%AE%E9%A2%98/" style="font-size: 10px;">字符问题</a> <a href="/tags/%E5%AD%A6%E4%B9%A0%E6%96%B9%E6%B3%95/" style="font-size: 10px;">学习方法</a> <a href="/tags/%E5%AE%89%E5%8D%93/" style="font-size: 10px;">安卓</a> <a href="/tags/%E5%AE%9E%E7%94%A8/" style="font-size: 10px;">实用</a> <a href="/tags/%E5%B0%81%E9%9D%A2/" style="font-size: 10px;">封面</a> <a href="/tags/%E5%B4%94%E5%BA%86%E6%89%8D/" style="font-size: 18.75px;">崔庆才</a> <a href="/tags/%E5%B7%A5%E5%85%B7/" style="font-size: 12.5px;">工具</a> <a href="/tags/%E5%BC%80%E5%8F%91%E5%B7%A5%E5%85%B7/" style="font-size: 10px;">开发工具</a> <a href="/tags/%E5%BE%AE%E8%BD%AF/" style="font-size: 10px;">微软</a> <a href="/tags/%E6%80%9D%E8%80%83/" style="font-size: 10px;">思考</a> <a href="/tags/%E6%89%8B%E6%9C%BA%E8%AE%BF%E9%97%AE/" style="font-size: 10px;">手机访问</a> <a href="/tags/%E6%95%99%E7%A8%8B/" style="font-size: 10px;">教程</a> <a href="/tags/%E6%95%99%E8%82%B2/" style="font-size: 10px;">教育</a> <a href="/tags/%E6%96%B0%E4%B9%A6/" style="font-size: 12.5px;">新书</a> <a href="/tags/%E6%96%B9%E6%B3%95%E8%AE%BA/" style="font-size: 10px;">方法论</a> <a href="/tags/%E6%97%85%E6%B8%B8/" style="font-size: 10px;">旅游</a> <a href="/tags/%E6%97%A5%E5%BF%97/" style="font-size: 10px;">日志</a> <a href="/tags/%E6%9A%97%E6%97%B6%E9%97%B4/" style="font-size: 10px;">暗时间</a> <a href="/tags/%E6%9D%9C%E5%85%B0%E7%89%B9/" style="font-size: 11.25px;">杜兰特</a> <a href="/tags/%E6%A1%8C%E9%9D%A2/" style="font-size: 10px;">桌面</a> <a href="/tags/%E6%AD%8C%E5%8D%95/" style="font-size: 10px;">歌单</a> <a href="/tags/%E6%B1%9F%E5%8D%97/" style="font-size: 10px;">江南</a> <a href="/tags/%E6%B8%B8%E6%88%8F/" style="font-size: 10px;">游戏</a> <a href="/tags/%E7%84%A6%E8%99%91/" style="font-size: 10px;">焦虑</a> <a href="/tags/%E7%88%AC%E8%99%AB/" style="font-size: 16.25px;">爬虫</a> <a href="/tags/%E7%88%AC%E8%99%AB%E4%B9%A6%E7%B1%8D/" style="font-size: 11.25px;">爬虫书籍</a> <a href="/tags/%E7%8E%AF%E5%A2%83%E5%8F%98%E9%87%8F/" style="font-size: 10px;">环境变量</a> <a href="/tags/%E7%94%9F%E6%B4%BB%E7%AC%94%E8%AE%B0/" style="font-size: 10px;">生活笔记</a> <a href="/tags/%E7%99%BB%E5%BD%95/" style="font-size: 10px;">登录</a> <a href="/tags/%E7%9F%A5%E4%B9%8E/" style="font-size: 10px;">知乎</a> <a href="/tags/%E7%9F%AD%E4%BF%A1/" style="font-size: 10px;">短信</a> <a href="/tags/%E7%9F%AD%E4%BF%A1%E9%AA%8C%E8%AF%81%E7%A0%81/" style="font-size: 10px;">短信验证码</a> <a href="/tags/%E7%AC%94%E8%AE%B0%E8%BD%AF%E4%BB%B6/" style="font-size: 10px;">笔记软件</a> <a href="/tags/%E7%AF%AE%E7%BD%91/" style="font-size: 10px;">篮网</a> <a href="/tags/%E7%BA%B8%E5%BC%A0/" style="font-size: 10px;">纸张</a> <a href="/tags/%E7%BB%84%E4%BB%B6/" style="font-size: 10px;">组件</a> <a href="/tags/%E7%BD%91%E7%AB%99/" style="font-size: 10px;">网站</a> <a href="/tags/%E7%BD%91%E7%BB%9C%E7%88%AC%E8%99%AB/" style="font-size: 11.25px;">网络爬虫</a> <a href="/tags/%E7%BE%8E%E5%AD%A6/" style="font-size: 10px;">美学</a> <a href="/tags/%E8%82%89%E5%A4%B9%E9%A6%8D/" style="font-size: 10px;">肉夹馍</a> <a href="/tags/%E8%85%BE%E8%AE%AF%E4%BA%91/" style="font-size: 10px;">腾讯云</a> <a href="/tags/%E8%87%AA%E5%BE%8B/" style="font-size: 10px;">自律</a> <a href="/tags/%E8%A5%BF%E5%B0%91%E7%88%B7/" style="font-size: 10px;">西少爷</a> <a href="/tags/%E8%A7%86%E9%A2%91/" style="font-size: 10px;">视频</a> <a href="/tags/%E8%B0%B7%E6%AD%8C%E9%AA%8C%E8%AF%81%E7%A0%81/" style="font-size: 10px;">谷歌验证码</a> <a href="/tags/%E8%BF%90%E8%90%A5/" style="font-size: 10px;">运营</a> <a href="/tags/%E8%BF%9C%E7%A8%8B/" style="font-size: 10px;">远程</a> <a href="/tags/%E9%80%86%E5%90%91/" style="font-size: 10px;">逆向</a> <a href="/tags/%E9%85%8D%E7%BD%AE/" style="font-size: 10px;">配置</a> <a href="/tags/%E9%87%8D%E8%A3%85/" style="font-size: 10px;">重装</a> <a href="/tags/%E9%98%BF%E6%9D%9C/" style="font-size: 10px;">阿杜</a> <a href="/tags/%E9%9D%99%E8%A7%85/" style="font-size: 17.5px;">静觅</a> <a href="/tags/%E9%A2%A0%E8%A6%86/" style="font-size: 10px;">颠覆</a> <a href="/tags/%E9%A3%9E%E4%BF%A1/" style="font-size: 10px;">飞信</a> <a href="/tags/%E9%B8%BF%E8%92%99/" style="font-size: 10px;">鸿蒙</a>
              </div>
              <script>
                const tagsColors = ['#00a67c', '#5cb85c', '#d9534f', '#567e95', '#b37333', '#f4843d', '#15a287']
                const tagsElements = document.querySelectorAll('.sidebar-panel-tags .content a')
                tagsElements.forEach((item) =>
                {
                  item.style.backgroundColor = tagsColors[Math.floor(Math.random() * tagsColors.length)]
                })

              </script>
            </div>
            <div class="sidebar-panel sidebar-panel-categories sidebar-panel-active">
              <h4 class="name"> 分类 </h4>
              <div class="content">
                <ul class="category-list">
                  <li class="category-list-item"><a class="category-list-link" href="/categories/C-C/">C/C++</a><span class="category-list-count">23</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/HTML/">HTML</a><span class="category-list-count">14</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Java/">Java</a><span class="category-list-count">5</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/JavaScript/">JavaScript</a><span class="category-list-count">26</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Linux/">Linux</a><span class="category-list-count">15</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Markdown/">Markdown</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Net/">Net</a><span class="category-list-count">4</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Other/">Other</a><span class="category-list-count">39</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/PHP/">PHP</a><span class="category-list-count">27</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Paper/">Paper</a><span class="category-list-count">2</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Python/">Python</a><span class="category-list-count">261</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/TypeScript/">TypeScript</a><span class="category-list-count">2</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E5%B1%95%E7%A4%BA/">个人展示</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E6%97%A5%E8%AE%B0/">个人日记</a><span class="category-list-count">9</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E8%AE%B0%E5%BD%95/">个人记录</a><span class="category-list-count">4</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E9%9A%8F%E7%AC%94/">个人随笔</a><span class="category-list-count">15</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E5%AE%89%E8%A3%85%E9%85%8D%E7%BD%AE/">安装配置</a><span class="category-list-count">59</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E6%8A%80%E6%9C%AF%E6%9D%82%E8%B0%88/">技术杂谈</a><span class="category-list-count">88</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E6%9C%AA%E5%88%86%E7%B1%BB/">未分类</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E7%94%9F%E6%B4%BB%E7%AC%94%E8%AE%B0/">生活笔记</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E7%A6%8F%E5%88%A9%E4%B8%93%E5%8C%BA/">福利专区</a><span class="category-list-count">6</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E8%81%8C%E4%BD%8D%E6%8E%A8%E8%8D%90/">职位推荐</a><span class="category-list-count">2</span></li>
                </ul>
              </div>
            </div>
            <div class="sidebar-panel sidebar-panel-friends sidebar-panel-active">
              <h4 class="name"> 友情链接 </h4>
              <ul class="friends">
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/j2dub.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.findhao.net/" target="_blank" rel="noopener">FindHao</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ou6mm.jpg">
                  </span>
                  <span class="link">
                    <a href="https://diygod.me/" target="_blank" rel="noopener">DIYgod</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/6apxu.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.51dev.com/" target="_blank" rel="noopener">IT技术社区</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.jankl.com/img/titleshu.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.jankl.com/" target="_blank" rel="noopener">liberalist</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/bqlbs.png">
                  </span>
                  <span class="link">
                    <a href="http://www.urselect.com/" target="_blank" rel="noopener">优社电商</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/8s88c.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.yuanrenxue.com/" target="_blank" rel="noopener">猿人学</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/2wgg5.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.yunlifang.cn/" target="_blank" rel="noopener">云立方</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/shwr6.png">
                  </span>
                  <span class="link">
                    <a href="http://lanbing510.info/" target="_blank" rel="noopener">冰蓝</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/blvoh.jpg">
                  </span>
                  <span class="link">
                    <a href="https://lengyue.me/" target="_blank" rel="noopener">冷月</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="http://qianxunclub.com/favicon.png">
                  </span>
                  <span class="link">
                    <a href="http://qianxunclub.com/" target="_blank" rel="noopener">千寻啊千寻</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/0044u.jpg">
                  </span>
                  <span class="link">
                    <a href="http://kodcloud.com/" target="_blank" rel="noopener">可道云</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ygnpn.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.kunkundashen.cn/" target="_blank" rel="noopener">坤坤大神</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/22uv1.png">
                  </span>
                  <span class="link">
                    <a href="http://www.cenchong.com/" target="_blank" rel="noopener">岑冲博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ev9kl.png">
                  </span>
                  <span class="link">
                    <a href="http://www.zxiaoji.com/" target="_blank" rel="noopener">张小鸡</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.503error.com/favicon.ico">
                  </span>
                  <span class="link">
                    <a href="https://www.503error.com/" target="_blank" rel="noopener">张志明个人博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/x714o.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.hubwiz.com/" target="_blank" rel="noopener">汇智网</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/129d8.png">
                  </span>
                  <span class="link">
                    <a href="https://www.bysocket.com/" target="_blank" rel="noopener">泥瓦匠BYSocket</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.xiongge.club/favicon.ico">
                  </span>
                  <span class="link">
                    <a href="https://www.xiongge.club/" target="_blank" rel="noopener">熊哥club</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/3w4fe.png">
                  </span>
                  <span class="link">
                    <a href="https://zerlong.com/" target="_blank" rel="noopener">知语</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/44hxf.png">
                  </span>
                  <span class="link">
                    <a href="http://redstonewill.com/" target="_blank" rel="noopener">红色石头</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/8g1fk.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.laodong.me/" target="_blank" rel="noopener">老董博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/wkaus.jpg">
                  </span>
                  <span class="link">
                    <a href="https://zhaoshuai.me/" target="_blank" rel="noopener">碎念</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/pgo0r.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.chenwenguan.com/" target="_blank" rel="noopener">陈文管的博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/kk82a.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.lxlinux.net/" target="_blank" rel="noopener">良许Linux教程网</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/lj0t2.jpg">
                  </span>
                  <span class="link">
                    <a href="https://tanqingbo.cn/" target="_blank" rel="noopener">IT码农</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/i8cdr.png">
                  </span>
                  <span class="link">
                    <a href="https://junyiseo.com/" target="_blank" rel="noopener">均益个人博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/chwv2.png">
                  </span>
                  <span class="link">
                    <a href="https://brucedone.com/" target="_blank" rel="noopener">大鱼的鱼塘</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/2y43o.png">
                  </span>
                  <span class="link">
                    <a href="http://bbs.nightteam.cn/" target="_blank" rel="noopener">夜幕爬虫安全论坛</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/zvc3w.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.weishidong.com/" target="_blank" rel="noopener">韦世东的技术专栏</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ebudy.jpg">
                  </span>
                  <span class="link">
                    <a href="https://chuanjiabing.com/" target="_blank" rel="noopener">穿甲兵技术社区</a>
                  </span>
                </li>
              </ul>
            </div>
          </div>
        </aside>
        <div id="sidebar-dimmer"></div>
      </div>
    </main>
    <footer class="footer">
      <div class="footer-inner">
        <div class="copyright"> &copy; <span itemprop="copyrightYear">2021</span>
          <span class="with-love">
            <i class="fa fa-heart"></i>
          </span>
          <span class="author" itemprop="copyrightHolder">崔庆才丨静觅</span>
          <span class="post-meta-divider">|</span>
          <span class="post-meta-item-icon">
            <i class="fa fa-chart-area"></i>
          </span>
          <span title="站点总字数">2.6m</span>
          <span class="post-meta-divider">|</span>
          <span class="post-meta-item-icon">
            <i class="fa fa-coffee"></i>
          </span>
          <span title="站点阅读时长">39:54</span>
        </div>
        <div class="powered-by">由 <a href="https://hexo.io/" class="theme-link" rel="noopener" target="_blank">Hexo</a> & <a href="https://pisces.theme-next.org/" class="theme-link" rel="noopener" target="_blank">NexT.Pisces</a> 强力驱动 </div>
        <div class="beian"><a href="https://beian.miit.gov.cn/" rel="noopener" target="_blank">京ICP备18015597号-1 </a>
        </div>
        <script>
          (function ()
          {
            function leancloudSelector(url)
            {
              url = encodeURI(url);
              return document.getElementById(url).querySelector('.leancloud-visitors-count');
            }

            function addCount(Counter)
            {
              var visitors = document.querySelector('.leancloud_visitors');
              var url = decodeURI(visitors.id);
              var title = visitors.dataset.flagTitle;
              Counter('get', '/classes/Counter?where=' + encodeURIComponent(JSON.stringify(
              {
                url
              }))).then(response => response.json()).then((
              {
                results
              }) =>
              {
                if (results.length > 0)
                {
                  var counter = results[0];
                  leancloudSelector(url).innerText = counter.time + 1;
                  Counter('put', '/classes/Counter/' + counter.objectId,
                  {
                    time:
                    {
                      '__op': 'Increment',
                      'amount': 1
                    }
                  }).catch(error =>
                  {
                    console.error('Failed to save visitor count', error);
                  });
                }
                else
                {
                  Counter('post', '/classes/Counter',
                  {
                    title,
                    url,
                    time: 1
                  }).then(response => response.json()).then(() =>
                  {
                    leancloudSelector(url).innerText = 1;
                  }).catch(error =>
                  {
                    console.error('Failed to create', error);
                  });
                }
              }).catch(error =>
              {
                console.error('LeanCloud Counter Error', error);
              });
            }

            function showTime(Counter)
            {
              var visitors = document.querySelectorAll('.leancloud_visitors');
              var entries = [...visitors].map(element =>
              {
                return decodeURI(element.id);
              });
              Counter('get', '/classes/Counter?where=' + encodeURIComponent(JSON.stringify(
              {
                url:
                {
                  '$in': entries
                }
              }))).then(response => response.json()).then((
              {
                results
              }) =>
              {
                for (let url of entries)
                {
                  let target = results.find(item => item.url === url);
                  leancloudSelector(url).innerText = target ? target.time : 0;
                }
              }).catch(error =>
              {
                console.error('LeanCloud Counter Error', error);
              });
            }
            let
            {
              app_id,
              app_key,
              server_url
            } = {
              "enable": true,
              "app_id": "6X5dRQ0pnPWJgYy8SXOg0uID-gzGzoHsz",
              "app_key": "ziLDVEy73ne5HtFTiGstzHMS",
              "server_url": "https://6x5drq0p.lc-cn-n1-shared.com",
              "security": false
            };

            function fetchData(api_server)
            {
              var Counter = (method, url, data) =>
              {
                return fetch(`${api_server}/1.1${url}`,
                {
                  method,
                  headers:
                  {
                    'X-LC-Id': app_id,
                    'X-LC-Key': app_key,
                    'Content-Type': 'application/json',
                  },
                  body: JSON.stringify(data)
                });
              };
              if (CONFIG.page.isPost)
              {
                if (CONFIG.hostname !== location.hostname) return;
                addCount(Counter);
              }
              else if (document.querySelectorAll('.post-title-link').length >= 1)
              {
                showTime(Counter);
              }
            }
            let api_server = app_id.slice(-9) !== '-MdYXbMMI' ? server_url : `https://${app_id.slice(0, 8).toLowerCase()}.api.lncldglobal.com`;
            if (api_server)
            {
              fetchData(api_server);
            }
            else
            {
              fetch('https://app-router.leancloud.cn/2/route?appId=' + app_id).then(response => response.json()).then((
              {
                api_server
              }) =>
              {
                fetchData('https://' + api_server);
              });
            }
          })();

        </script>
      </div>
      <div class="footer-stat">
        <span id="cnzz_stat_icon_1279355174"></span>
        <script type="text/javascript">
          document.write(unescape("%3Cspan id='cnzz_stat_icon_1279355174'%3E%3C/span%3E%3Cscript src='https://v1.cnzz.com/z_stat.php%3Fid%3D1279355174%26online%3D1%26show%3Dline' type='text/javascript'%3E%3C/script%3E"));

        </script>
      </div>
    </footer>
  </div>
  <script src="//cdn.jsdelivr.net/npm/animejs@3.2.1/lib/anime.min.js"></script>
  <script src="//cdn.jsdelivr.net/npm/pangu@4/dist/browser/pangu.min.js"></script>
  <script src="/js/utils.js"></script>
  <script src="/.js"></script>
  <script src="/js/schemes/pisces.js"></script>
  <script src="/.js"></script>
  <script src="/js/next-boot.js"></script>
  <script src="/.js"></script>
  <script>
    (function ()
    {
      var canonicalURL, curProtocol;
      //Get the <link> tag
      var x = document.getElementsByTagName("link");
      //Find the last canonical URL
      if (x.length > 0)
      {
        for (i = 0; i < x.length; i++)
        {
          if (x[i].rel.toLowerCase() == 'canonical' && x[i].href)
          {
            canonicalURL = x[i].href;
          }
        }
      }
      //Get protocol
      if (!canonicalURL)
      {
        curProtocol = window.location.protocol.split(':')[0];
      }
      else
      {
        curProtocol = canonicalURL.split(':')[0];
      }
      //Get current URL if the canonical URL does not exist
      if (!canonicalURL) canonicalURL = window.location.href;
      //Assign script content. Replace current URL with the canonical URL
      ! function ()
      {
        var e = /([http|https]:\/\/[a-zA-Z0-9\_\.]+\.baidu\.com)/gi,
          r = canonicalURL,
          t = document.referrer;
        if (!e.test(r))
        {
          var n = (String(curProtocol).toLowerCase() === 'https') ? "https://sp0.baidu.com/9_Q4simg2RQJ8t7jm9iCKT-xh_/s.gif" : "//api.share.baidu.com/s.gif";
          t ? (n += "?r=" + encodeURIComponent(document.referrer), r && (n += "&l=" + r)) : r && (n += "?l=" + r);
          var i = new Image;
          i.src = n
        }
      }(window);
    })();

  </script>
  <script src="/js/local-search.js"></script>
  <script src="/.js"></script>
</body>

</html>
